Moving stuff around.

This commit is contained in:
Kiri 2024-07-14 10:23:19 -07:00
parent 5d3b74d798
commit bdd56c30d1
19 changed files with 293 additions and 232 deletions

View File

@ -3,15 +3,11 @@ extends Node
func _ready():
var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
bw._unpack_python()
bw.unpack_python()
##_unpack_python()
#print(_get_runtime_python_executable_godot_path())
#print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()))
#
var out = []
var ret = OS.execute(
ProjectSettings.globalize_path(bw._get_runtime_python_executable_godot_path()),
bw.get_runtime_python_executable_system_path(),
["--version"], out, true)
print("Ret: ", ret)

View File

@ -14,19 +14,17 @@ func _export_begin(
var arch_list = []
if "linux" in features:
#platform_list.append(build_wrangler._get_python_platform("Linux"))
platform_list.append("Linux")
if "windows" in features:
platform_list.append("Windows")
if "x86_64" in features:
arch_list.append("x86_64")
# TODO: Other platforms (macos)
for platform in platform_list:
for arch in arch_list:
var python_arch : String = build_wrangler._get_python_architecture(arch)
var python_platform : String = build_wrangler._get_python_platform(platform, arch)
var archive_to_export = build_wrangler.get_export_python_archive_godot_path(python_platform, python_arch)
#print("EXPORT ME TOO: ", archive_to_export)
var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch)
var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export)
print("Adding file: ", archive_to_export, " ", len(file_contents))
add_file(archive_to_export, file_contents, false)

View File

@ -0,0 +1,233 @@
extends RefCounted
class_name KiriPythonBuildWrangler
# Cached release info so we don't have to constantly reload the .json file.
var _python_release_info : Dictionary = {}
#region releaseinfo file interactions
func _get_python_release_info():
if _python_release_info == {}:
var this_script_path = get_script().resource_path
var this_script_dir = this_script_path.get_base_dir()
var release_info_path = this_script_dir.path_join("StandalonePythonBuilds/python_release_info.json")
_python_release_info = load(release_info_path).data
# If you hit this assert, your python_release_info.json file is probably
# missing and you missed a setup step. Check the README.
assert(_python_release_info != null)
return _python_release_info
func _get_python_version():
var info = _get_python_release_info()
var versions : Array = info["versions"]
# Sort version numbers so that the highest version is the first element.
versions.sort_custom(func(a : String, b : String):
var version_parts_a : PackedStringArray = a.split(".")
var version_parts_b : PackedStringArray = b.split(".")
for i in range(0, 3):
if int(version_parts_a[i]) > int(version_parts_b[i]):
return true
if int(version_parts_a[i]) < int(version_parts_b[i]):
return false
return false)
return versions[0]
func _get_python_release() -> String:
var info = _get_python_release_info()
return info["release"]
#endregion
#region Python archive filename wrangling
# Generate the archive filename based on what we've figured out from the release
# info, the platform, architecture, optimizations, and so on. This is just the
# filename, not including the full path.
#
# Use _generate_python_archive_full_path() to generate the full path (as a
# res:// path).
func _generate_python_archive_string(
python_version : String,
python_release : String,
arch : String,
os : String,
opt : String) -> String:
return "cpython-{python_version}+{python_release}-{python_arch}-{python_os}-{python_opt}-full.tar.zip".format({
"python_version" : python_version,
"python_release" : python_release,
"python_arch" : arch,
"python_os" : os,
"python_opt" : opt
})
# Get full path (in Godot) to the archive for a given Python build.
func _generate_python_archive_full_path(
python_version : String,
python_release : String,
arch : String,
os : String,
opt : String) -> String:
var just_the_archive_filename = _generate_python_archive_string(
python_version, python_release, arch, os, opt)
var this_script_path = get_script().resource_path
var this_script_dir = this_script_path.get_base_dir()
var python_archive_path = this_script_dir.path_join(
"StandalonePythonBuilds").path_join(just_the_archive_filename)
return python_archive_path
# os_name as it appears in the Python archive filename.
func _get_python_opt_for_os(os_name : String) -> String:
if os_name == "pc-windows-msvc-shared":
return "pgo"
# TODO: (macos)
# Linux default.
return "pgo+lto"
# Note: arch variable is output of _get_python_architecture, not whatever Godot
# returns. os_name IS what Godot returns from OS.get_name().
func _get_python_platform(os_name : String, arch : String) -> String:
var os_name_mappings : Dictionary = {
"Linux" : "unknown-linux-gnu",
"macOS" : "apple-darwin", # TODO: Test this. (macos)
"Windows" : "pc-windows-msvc-shared"
}
# Special case for armv7 Linux:
if arch == "armv7" and os_name == "Linux":
return "linux-gnueabi"
assert(os_name_mappings.has(os_name))
return os_name_mappings[os_name]
func _get_python_architecture(engine_arch : String) -> String:
var arch_name_mappings : Dictionary = {
"x86_64" : "x86_64",
"x86_32" : "i686",
"arm64" : "aarch64", # FIXME: I dunno if this is correct.
"arm32" : "armv7", # FIXME: I dunno if this is correct.
}
assert(arch_name_mappings.has(engine_arch))
return arch_name_mappings[engine_arch]
func _detect_archive_for_runtime() -> String:
var python_version : String = _get_python_version()
var python_release : String = _get_python_release()
var arch : String = _get_python_architecture(Engine.get_architecture_name())
var os_name : String = _get_python_platform(OS.get_name(), arch)
var opt = _get_python_opt_for_os(os_name)
return _generate_python_archive_full_path(
python_version, python_release,
arch, os_name, opt)
# Params are Godot's names for OSes and architectures (eg "Windows", "Linux",
# etc), not Python archive filename fields. Use things like OS.get_name().
func _detect_archive_for_build(
os_name_from_godot : String,
arch_from_godot : String) -> String:
var python_version : String = _get_python_version()
var python_release : String = _get_python_release()
var arch : String = _get_python_architecture(arch_from_godot)
var os_name : String = _get_python_platform(os_name_from_godot, arch)
var opt = _get_python_opt_for_os(os_name)
return _generate_python_archive_full_path(
python_version, python_release,
arch, os_name, opt)
#endregion
#region Cache path wrangling
# Get the cache path, relative to the user data dir.
# Example return value:
# "_python_dist/20240415/3.12.3"
func _get_cache_path_relative():
return "_python_dist".path_join(_get_python_release()).path_join(_get_python_version())
# Get the full cache path, as understood by the OS.
# Example return value:
# "/home/kiri/.local/share/godot/app_userdata/GodotJSONRPCTest/_python_dist/20240415/3.12.3"
func _get_cache_path_system() -> String:
return OS.get_user_data_dir().path_join(_get_cache_path_relative())
# Get the full cache path, as understood by Godot.
# Example return value:
# "user://_python_dist/20240415/3.12.3"
func _get_cache_path_godot() -> String:
return "user://".path_join(_get_cache_path_relative())
#endregion
#region Public API
# Get the expected path to the Python executable. This is where we think it'll
# end up, not where it actually did end up. This can be called without actually
# extracting the archive. In fact, we need it to act that way because we use it
# to determine if there's already a Python install in-place.
#
# Path is a Godot path. Use ProjectSettings.globalize_path() to conver to a
# system path.
#
# Example return:
# "user://_python_dist/20240415/3.12.3/python/install/bin/python3"
func get_runtime_python_executable_godot_path() -> String:
var base_dir = _get_cache_path_godot().path_join("python/install")
if OS.get_name() == "Windows":
return base_dir.path_join("python.exe")
else:
return base_dir.path_join("bin/python3")
# TODO: Other platforms (macos).
# Get system path for the Python executable, which is what we actually need to
# use to execute it in most cases.
#
# Example return:
# "home/<user>/.local/share/godot/app_userdata/<project>/_python_dist/20240415/3.12.3/python/install/bin/python3"
func get_runtime_python_executable_system_path() -> String:
return ProjectSettings.globalize_path(get_runtime_python_executable_godot_path())
func unpack_python(overwrite : bool = false):
var cache_path_godot : String = _get_cache_path_godot()
# Check to see if the Python executable already exists. If it does, we might
# just skip unpacking.
var python_executable_expected_path : String = \
get_runtime_python_executable_godot_path()
if not overwrite:
if FileAccess.file_exists(python_executable_expected_path):
return
# Open archive.
var python_archive_path : String = _detect_archive_for_runtime()
var reader : TARReader = TARReader.new()
var err : Error = reader.open(python_archive_path)
assert(err == OK)
# Get files.
var file_list : PackedStringArray = reader.get_files()
# Extract files.
for relative_filename : String in file_list:
reader.unpack_file(cache_path_godot, relative_filename)
# TODO: Clear cache function. Uninstall Python, etc.
#endregion

View File

@ -10,7 +10,7 @@
# DO NOT USE THIS ON UNTRUSTED DATA.
extends RefCounted
class_name TARReader
class_name KiriTARReader
#region Internal data
@ -38,6 +38,10 @@ var _internal_file_list = []
var _reader : ZIPReader = null
var _tar_file_cache : PackedByteArray = []
func _load_record(record : TarFileRecord) -> PackedByteArray:
load_cache()
return _tar_file_cache.slice(record.offset, record.offset + record.file_size)
#endregion
#region Cache wrangling
@ -69,6 +73,29 @@ func load_cache() -> Error:
#endregion
#region Number wrangling
func _octal_str_to_int(s : String) -> int:
var ret : int = 0;
var digit_multiplier = 1;
while len(s):
var lsb = s.substr(len(s) - 1, 1)
s = s.substr(0, len(s) - 1)
ret += digit_multiplier * lsb.to_int()
digit_multiplier *= 8
return ret
func _pad_to_512(x : int) -> int:
var x_lowbits = x & 511
var x_hibits = x & ~511
if x_lowbits:
x_hibits += 512
return x_hibits
#endregion
func close() -> Error:
_internal_file_list = []
_reader.close()
@ -92,25 +119,6 @@ func get_files() -> PackedStringArray:
ret.append(record.filename)
return ret
func _octal_str_to_int(s : String) -> int:
var ret : int = 0;
var digit_multiplier = 1;
while len(s):
var lsb = s.substr(len(s) - 1, 1)
s = s.substr(0, len(s) - 1)
ret += digit_multiplier * lsb.to_int()
digit_multiplier *= 8
return ret
func _pad_to_512(x : int) -> int:
var x_lowbits = x & 511
var x_hibits = x & ~511
if x_lowbits:
x_hibits += 512
return x_hibits
func open(path: String) -> Error:
assert(not _reader)
@ -234,10 +242,7 @@ func open(path: String) -> Error:
return OK
func _load_record(record : TarFileRecord) -> PackedByteArray:
load_cache()
return _tar_file_cache.slice(record.offset, record.offset + record.file_size)
# Extract a file into memory as a PackedByteArray.
func read_file(path : String, case_sensitive : bool = true) -> PackedByteArray:
for record : TarFileRecord in _internal_file_list:
@ -250,7 +255,14 @@ func read_file(path : String, case_sensitive : bool = true) -> PackedByteArray:
return []
func unpack_file(dest_path : String, filename : String):
# Extract a file to a specific path. Sets permissions when possible, handles
# symlinks and directories. Will extract to the dest_path plus the internal
# relative path.
#
# Example:
# dest_path: "foo/bar", filename: "butts/whatever/thingy.txt"
# extracts to: "foo/bar/butts/whatever/thingy.txt"
func unpack_file(dest_path : String, filename : String, overwrite : bool = false):
var full_dest_path : String = dest_path.path_join(filename)
DirAccess.make_dir_recursive_absolute(full_dest_path.get_base_dir())
@ -269,6 +281,12 @@ func unpack_file(dest_path : String, filename : String):
# traversal attacks.
if record.filename == filename:
# FIXME: Somehow this is slower than just overwriting the file.
# Awesome. /s
if overwrite == false and FileAccess.file_exists(full_dest_path):
continue
if record.is_link:
# Okay, look. I know that symbolic links technically exist on
@ -276,24 +294,30 @@ func unpack_file(dest_path : String, filename : String):
# if for some reason you need to support that. -Kiri
assert(OS.get_name() != "Windows")
# Fire off a command to make a symbolic link on *normal* OSes.
var err = OS.execute("ln", [
"-s",
record.link_destination,
ProjectSettings.globalize_path(full_dest_path) ])
ProjectSettings.globalize_path(full_dest_path)
])
assert(err != -1)
elif record.is_directory:
# It's just a directory. Make it.
DirAccess.make_dir_recursive_absolute(full_dest_path)
else:
# Okay this is an actual file. Extract it.
var file_data : PackedByteArray = read_file(record.filename)
var out_file = FileAccess.open(full_dest_path, FileAccess.WRITE)
out_file.store_buffer(file_data)
out_file.close()
# Set permissions.
# Set permissions (on normal OSes, not Windows). I don't think this
# applies to symlinks, though.
if not record.is_link:
if OS.get_name() != "Windows":
var err = OS.execute("chmod", [

View File

@ -1,190 +0,0 @@
extends RefCounted
class_name KiriPythonBuildWrangler
var _python_release_info : Dictionary = {}
func _get_python_release_info():
if _python_release_info == {}:
var this_script_path = get_script().resource_path
var this_script_dir = this_script_path.get_base_dir()
var release_info_path = this_script_dir.path_join("StandalonePythonBuilds/python_release_info.json")
_python_release_info = load(release_info_path).data
return _python_release_info
func _get_python_version():
var info = _get_python_release_info()
var versions : Array = info["versions"]
# Sort version numbers so that the highest version is the first element.
versions.sort_custom(func(a : String, b : String):
var version_parts_a : PackedStringArray = a.split(".")
var version_parts_b : PackedStringArray = b.split(".")
for i in range(0, 3):
if int(version_parts_a[i]) > int(version_parts_b[i]):
return true
if int(version_parts_a[i]) < int(version_parts_b[i]):
return false
return false)
return versions[0]
func _get_python_release():
var info = _get_python_release_info()
return info["release"]
func _generate_python_archive_string(
python_version : String,
python_release : String,
arch : String,
os : String,
opt : String):
return "cpython-{python_version}+{python_release}-{python_arch}-{python_os}-{python_opt}-full.tar.zip".format(
{
"python_version" : python_version,
"python_release" : python_release,
"python_arch" : arch,
"python_os" : os,
"python_opt" : opt
})
func _get_python_opt_for_os(os_name : String) -> String:
if os_name == "pc-windows-msvc-shared":
return "pgo"
return "pgo+lto"
func _detect_archive_for_runtime( \
python_version : String,
python_release : String):
var arch : String = _get_python_architecture(Engine.get_architecture_name())
var os_name : String = _get_python_platform(OS.get_name(), arch)
var opt = _get_python_opt_for_os(os_name)
var archive_str : String = _generate_python_archive_string(
python_version, python_release,
arch, os_name, opt)
return archive_str
func _detect_archive_for_build(
python_version : String,
python_release : String,
arch : String,
os_name : String):
var opt = _get_python_opt_for_os(os_name)
var archive_str : String = _generate_python_archive_string(
python_version, python_release,
arch, os_name, opt)
return archive_str
# Note: arch variable is output of _get_python_architecture, not whatever Godot
# returns.
func _get_python_platform(os_name : String, arch : String) -> String:
var os_name_mappings : Dictionary = {
"Linux" : "unknown-linux-gnu",
"macOS" : "apple-darwin",
"Windows" : "pc-windows-msvc-shared"
}
# Special case for armv7 Linux:
if arch == "armv7" and os_name == "Linux":
return "linux-gnueabi"
assert(os_name_mappings.has(os_name))
return os_name_mappings[os_name]
func _get_python_architecture(engine_arch : String) -> String:
var arch_name_mappings : Dictionary = {
"x86_64" : "x86_64",
"x86_32" : "i686",
"arm64" : "aarch64", # FIXME: I dunno if this is correct.
"arm32" : "armv7", # FIXME: I dunno if this is correct.
}
assert(arch_name_mappings.has(engine_arch))
return arch_name_mappings[engine_arch]
func _get_cache_path_relative():
return "_python_dist".path_join(_get_python_release()).path_join(_get_python_version())
func _get_cache_path_system():
return OS.get_user_data_dir().path_join(_get_cache_path_relative())
func _get_cache_path_godot():
return "user://".path_join(_get_cache_path_relative())
func _get_runtime_python_archive_godot_path() -> String:
var this_script_path = get_script().resource_path
var this_script_dir = this_script_path.get_base_dir()
var python_archive_path = this_script_dir.path_join(
"StandalonePythonBuilds").path_join(
_detect_archive_for_runtime(
_get_python_version(),
_get_python_release()))
return python_archive_path
func get_export_python_archive_godot_path(platform : String, arch : String) -> String:
var this_script_path = get_script().resource_path
var this_script_dir = this_script_path.get_base_dir()
var python_archive_path = this_script_dir.path_join(
"StandalonePythonBuilds").path_join(
_detect_archive_for_build(
_get_python_version(),
_get_python_release(), arch, platform))
return python_archive_path
func _unpack_python():
var python_archive_path = _get_runtime_python_archive_godot_path()
var reader : TARReader = TARReader.new()
reader.open(python_archive_path)
#print(reader.get_files())
var file_list : PackedStringArray = reader.get_files()
for relative_filename : String in file_list:
reader.unpack_file(_get_cache_path_godot(), relative_filename)
pass
func _get_runtime_python_executable_godot_path():
var base_dir = _get_cache_path_godot().path_join("python/install")
if OS.get_name() == "Windows":
return base_dir.path_join("python.exe")
else:
return base_dir.path_join("bin/python3")
# TODO: Other platforms.
# Testing code follows...
#func _ready():
#
#print(_detect_archive_for_runtime(
#_get_python_version(), _get_python_release()))
#
#print(_get_cache_path_godot())
#print(_get_cache_path_system())
#print(_get_runtime_python_archive_godot_path())
#
##_unpack_python()
#print(_get_runtime_python_executable_godot_path())
#print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()))
#
#var out = []
#OS.execute(
#ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()),
#["asdfjknsdcjknsdcjknsdjkc"], out, true)
#print(out)
#
#