Moving stuff around.

TestPythonInExport.gd

@@ -3,15 +3,11 @@ extends Node
 func _ready():
 	var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
 	
-	bw._unpack_python()
+	bw.unpack_python()
 	
-		##_unpack_python()
-	#print(_get_runtime_python_executable_godot_path())
-	#print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()))
-	#
 	var out = []
 	var ret = OS.execute(
-		ProjectSettings.globalize_path(bw._get_runtime_python_executable_godot_path()),
+		bw.get_runtime_python_executable_system_path(),
 		["--version"], out, true)
 
 	print("Ret: ", ret)

addons/KiriPythonRPCWrapper/KiriPythonBuildExportPlugin.gd

@@ -14,19 +14,17 @@ func _export_begin(
 	var arch_list = []
 
 	if "linux" in features:
-		#platform_list.append(build_wrangler._get_python_platform("Linux"))
 		platform_list.append("Linux")
 	if "windows" in features:
 		platform_list.append("Windows")
 	if "x86_64" in features:
 		arch_list.append("x86_64")
 	
+	# TODO: Other platforms (macos)
+	
 	for platform in platform_list:
 		for arch in arch_list:
-			var python_arch : String = build_wrangler._get_python_architecture(arch)
+			var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch)
-			var python_platform : String = build_wrangler._get_python_platform(platform, arch)
-			var archive_to_export = build_wrangler.get_export_python_archive_godot_path(python_platform, python_arch)
-			#print("EXPORT ME TOO: ", archive_to_export)
 			var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export)
 			print("Adding file: ", archive_to_export, " ", len(file_contents))
 			add_file(archive_to_export, file_contents, false)

addons/KiriPythonRPCWrapper/KiriPythonBuildWrangler.gd

@@ -0,0 +1,233 @@
+extends RefCounted
+class_name KiriPythonBuildWrangler
+
+# Cached release info so we don't have to constantly reload the .json file.
+var _python_release_info : Dictionary = {}
+
+#region releaseinfo file interactions
+
+func _get_python_release_info():
+	
+	if _python_release_info == {}:
+		
+		var this_script_path = get_script().resource_path
+		var this_script_dir = this_script_path.get_base_dir()
+		var release_info_path = this_script_dir.path_join("StandalonePythonBuilds/python_release_info.json")
+		_python_release_info = load(release_info_path).data
+		
+		# If you hit this assert, your python_release_info.json file is probably
+		# missing and you missed a setup step. Check the README.
+		assert(_python_release_info != null)
+
+	return _python_release_info
+
+func _get_python_version():
+	
+	var info = _get_python_release_info()
+	var versions : Array = info["versions"]
+	
+	# Sort version numbers so that the highest version is the first element.
+	versions.sort_custom(func(a : String, b : String):
+		var version_parts_a : PackedStringArray = a.split(".")
+		var version_parts_b : PackedStringArray = b.split(".")
+		for i in range(0, 3):
+			if int(version_parts_a[i]) > int(version_parts_b[i]):
+				return true
+			if int(version_parts_a[i]) < int(version_parts_b[i]):
+				return false
+		return false)
+
+	return versions[0]
+
+func _get_python_release() -> String:
+	var info = _get_python_release_info()
+	return info["release"]
+
+#endregion
+
+#region Python archive filename wrangling
+
+# Generate the archive filename based on what we've figured out from the release
+# info, the platform, architecture, optimizations, and so on. This is just the
+# filename, not including the full path.
+#
+# Use _generate_python_archive_full_path() to generate the full path (as a
+# res:// path).
+func _generate_python_archive_string(
+	python_version : String,
+	python_release : String,
+	arch : String,
+	os : String,
+	opt : String) -> String:
+
+	return "cpython-{python_version}+{python_release}-{python_arch}-{python_os}-{python_opt}-full.tar.zip".format({
+		"python_version" : python_version,
+		"python_release" : python_release,
+		"python_arch" : arch,
+		"python_os" : os,
+		"python_opt" : opt
+	})
+
+# Get full path (in Godot) to the archive for a given Python build.
+func _generate_python_archive_full_path(
+	python_version : String,
+	python_release : String,
+	arch : String,
+	os : String,
+	opt : String) -> String:
+
+	var just_the_archive_filename = _generate_python_archive_string(
+		python_version, python_release, arch, os, opt)
+
+	var this_script_path = get_script().resource_path
+	var this_script_dir = this_script_path.get_base_dir()
+	var python_archive_path = this_script_dir.path_join(
+		"StandalonePythonBuilds").path_join(just_the_archive_filename)
+	
+	return python_archive_path
+
+# os_name as it appears in the Python archive filename.
+func _get_python_opt_for_os(os_name : String) -> String:
+	if os_name == "pc-windows-msvc-shared":
+		return "pgo"
+	
+	# TODO: (macos)
+	
+	# Linux default.
+	return "pgo+lto"
+
+# Note: arch variable is output of _get_python_architecture, not whatever Godot
+# returns. os_name IS what Godot returns from OS.get_name().
+func _get_python_platform(os_name : String, arch : String) -> String:
+	var os_name_mappings : Dictionary = {
+		"Linux" : "unknown-linux-gnu",
+		"macOS" : "apple-darwin", # TODO: Test this. (macos)
+		"Windows" : "pc-windows-msvc-shared"
+	}
+	
+	# Special case for armv7 Linux:
+	if arch == "armv7" and os_name == "Linux":
+		return "linux-gnueabi"
+	
+	assert(os_name_mappings.has(os_name))
+	return os_name_mappings[os_name]
+
+func _get_python_architecture(engine_arch : String) -> String:
+	var arch_name_mappings : Dictionary = {
+		"x86_64" : "x86_64",
+		"x86_32" : "i686",
+		"arm64" : "aarch64", # FIXME: I dunno if this is correct.
+		"arm32" : "armv7",  # FIXME: I dunno if this is correct.
+	}
+	assert(arch_name_mappings.has(engine_arch))
+	return arch_name_mappings[engine_arch]
+
+func _detect_archive_for_runtime() -> String:
+	var python_version : String = _get_python_version()
+	var python_release : String = _get_python_release()
+	var arch : String = _get_python_architecture(Engine.get_architecture_name())
+	var os_name : String = _get_python_platform(OS.get_name(), arch)
+	var opt = _get_python_opt_for_os(os_name)
+
+	return _generate_python_archive_full_path(
+		python_version, python_release,
+		arch, os_name, opt)
+
+# Params are Godot's names for OSes and architectures (eg "Windows", "Linux",
+# etc), not Python archive filename fields. Use things like OS.get_name().
+func _detect_archive_for_build(
+	os_name_from_godot : String,
+	arch_from_godot : String) -> String:
+
+	var python_version : String = _get_python_version()
+	var python_release : String = _get_python_release()
+	
+	var arch : String = _get_python_architecture(arch_from_godot)
+	var os_name : String = _get_python_platform(os_name_from_godot, arch)
+	
+	var opt = _get_python_opt_for_os(os_name)
+	
+	return _generate_python_archive_full_path(
+		python_version, python_release,
+		arch, os_name, opt)
+
+#endregion
+
+#region Cache path wrangling
+# Get the cache path, relative to the user data dir.
+# Example return value:
+#   "_python_dist/20240415/3.12.3"
+func _get_cache_path_relative():
+	return "_python_dist".path_join(_get_python_release()).path_join(_get_python_version())
+
+# Get the full cache path, as understood by the OS.
+# Example return value:
+#   "/home/kiri/.local/share/godot/app_userdata/GodotJSONRPCTest/_python_dist/20240415/3.12.3"
+func _get_cache_path_system() -> String:
+	return OS.get_user_data_dir().path_join(_get_cache_path_relative())
+
+# Get the full cache path, as understood by Godot.
+# Example return value:
+#   "user://_python_dist/20240415/3.12.3"
+func _get_cache_path_godot() -> String:
+	return "user://".path_join(_get_cache_path_relative())
+
+#endregion
+
+#region Public API
+
+# Get the expected path to the Python executable. This is where we think it'll
+# end up, not where it actually did end up. This can be called without actually
+# extracting the archive. In fact, we need it to act that way because we use it
+# to determine if there's already a Python install in-place.
+#
+# Path is a Godot path. Use ProjectSettings.globalize_path() to conver to a
+# system path.
+#
+# Example return:
+#   "user://_python_dist/20240415/3.12.3/python/install/bin/python3"
+func get_runtime_python_executable_godot_path() -> String:
+	var base_dir = _get_cache_path_godot().path_join("python/install")
+	if OS.get_name() == "Windows":
+		return base_dir.path_join("python.exe")
+	else:
+		return base_dir.path_join("bin/python3")
+	
+	# TODO: Other platforms (macos).
+
+# Get system path for the Python executable, which is what we actually need to
+# use to execute it in most cases.
+#
+# Example return:
+#   "home/<user>/.local/share/godot/app_userdata/<project>/_python_dist/20240415/3.12.3/python/install/bin/python3"
+func get_runtime_python_executable_system_path() -> String:
+	return ProjectSettings.globalize_path(get_runtime_python_executable_godot_path())
+
+func unpack_python(overwrite : bool = false):
+
+	var cache_path_godot : String = _get_cache_path_godot()
+
+	# Check to see if the Python executable already exists. If it does, we might
+	# just skip unpacking.
+	var python_executable_expected_path : String = \
+		get_runtime_python_executable_godot_path()
+	if not overwrite:
+		if FileAccess.file_exists(python_executable_expected_path):
+			return
+
+	# Open archive.
+	var python_archive_path : String = _detect_archive_for_runtime()
+	var reader : TARReader = TARReader.new()
+	var err : Error = reader.open(python_archive_path)
+	assert(err == OK)
+
+	# Get files.
+	var file_list : PackedStringArray = reader.get_files()
+
+	# Extract files.
+	for relative_filename : String in file_list:
+		reader.unpack_file(cache_path_godot, relative_filename)
+
+# TODO: Clear cache function. Uninstall Python, etc.
+
+#endregion

addons/KiriPythonRPCWrapper/KiriTARReader.gd

@@ -10,7 +10,7 @@
 # DO NOT USE THIS ON UNTRUSTED DATA.
 
 extends RefCounted
-class_name TARReader
+class_name KiriTARReader
 
 #region Internal data
 
@@ -38,6 +38,10 @@ var _internal_file_list = []
 var _reader : ZIPReader = null
 var _tar_file_cache : PackedByteArray = []
 
+func _load_record(record : TarFileRecord) -> PackedByteArray:
+	load_cache()
+	return _tar_file_cache.slice(record.offset, record.offset + record.file_size)
+
 #endregion
 
 #region Cache wrangling
@@ -69,6 +73,29 @@ func load_cache() -> Error:
 
 #endregion
 
+#region Number wrangling
+
+func _octal_str_to_int(s : String) -> int:
+	var ret : int = 0;
+	var digit_multiplier = 1;
+	while len(s):
+		var lsb = s.substr(len(s) - 1, 1)
+		s = s.substr(0, len(s) - 1)
+		ret += digit_multiplier * lsb.to_int()
+		digit_multiplier *= 8
+	return ret
+
+func _pad_to_512(x : int) -> int:
+	var x_lowbits = x & 511
+	var x_hibits = x & ~511
+	
+	if x_lowbits:
+		x_hibits += 512
+	
+	return x_hibits
+
+#endregion
+
 func close() -> Error:
 	_internal_file_list = []
 	_reader.close()
@@ -92,25 +119,6 @@ func get_files() -> PackedStringArray:
 		ret.append(record.filename)
 	return ret
 
-func _octal_str_to_int(s : String) -> int:
-	var ret : int = 0;
-	var digit_multiplier = 1;
-	while len(s):
-		var lsb = s.substr(len(s) - 1, 1)
-		s = s.substr(0, len(s) - 1)
-		ret += digit_multiplier * lsb.to_int()
-		digit_multiplier *= 8
-	return ret
-
-func _pad_to_512(x : int) -> int:
-	var x_lowbits = x & 511
-	var x_hibits = x & ~511
-	
-	if x_lowbits:
-		x_hibits += 512
-	
-	return x_hibits
-
 func open(path: String) -> Error:
 
 	assert(not _reader)
@@ -234,10 +242,7 @@ func open(path: String) -> Error:
 	
 	return OK
 
-func _load_record(record : TarFileRecord) -> PackedByteArray:
+# Extract a file into memory as a PackedByteArray.
-	load_cache()
-	return _tar_file_cache.slice(record.offset, record.offset + record.file_size)
-
 func read_file(path : String, case_sensitive : bool = true) -> PackedByteArray:
 
 	for record : TarFileRecord in _internal_file_list:
@@ -250,7 +255,14 @@ func read_file(path : String, case_sensitive : bool = true) -> PackedByteArray:
 
 	return []
 
-func unpack_file(dest_path : String, filename : String):
+# Extract a file to a specific path. Sets permissions when possible, handles
+# symlinks and directories. Will extract to the dest_path plus the internal
+# relative path.
+#
+# Example:
+#   dest_path: "foo/bar", filename: "butts/whatever/thingy.txt"
+#   extracts to: "foo/bar/butts/whatever/thingy.txt"
+func unpack_file(dest_path : String, filename : String, overwrite : bool = false):
 	var full_dest_path : String = dest_path.path_join(filename)
 	DirAccess.make_dir_recursive_absolute(full_dest_path.get_base_dir())
 	
@@ -269,6 +281,12 @@ func unpack_file(dest_path : String, filename : String):
 		#        traversal attacks.
 		
 		if record.filename == filename:
+			
+			# FIXME: Somehow this is slower than just overwriting the file.
+			#        Awesome. /s
+			if overwrite == false and FileAccess.file_exists(full_dest_path):
+				continue
+			
 			if record.is_link:
 
 				# Okay, look. I know that symbolic links technically exist on
@@ -276,24 +294,30 @@ func unpack_file(dest_path : String, filename : String):
 				# if for some reason you need to support that. -Kiri
 				assert(OS.get_name() != "Windows")
 
+				# Fire off a command to make a symbolic link on *normal* OSes.
 				var err = OS.execute("ln", [
 					"-s",
 					record.link_destination,
-					ProjectSettings.globalize_path(full_dest_path) ])
+					ProjectSettings.globalize_path(full_dest_path)
+				])
+
 				assert(err != -1)
 			
 			elif record.is_directory:
 
+				# It's just a directory. Make it.
 				DirAccess.make_dir_recursive_absolute(full_dest_path)
 
 			else:
 
+				# Okay this is an actual file. Extract it.
 				var file_data : PackedByteArray = read_file(record.filename)
 				var out_file = FileAccess.open(full_dest_path, FileAccess.WRITE)
 				out_file.store_buffer(file_data)
 				out_file.close()
 
-			# Set permissions.
+			# Set permissions (on normal OSes, not Windows). I don't think this
+			# applies to symlinks, though.
 			if not record.is_link:
 				if OS.get_name() != "Windows":
 					var err = OS.execute("chmod", [

addons/kiripythonrpcwrapper/PythonBuildWrangler.gd

@@ -1,190 +0,0 @@
-extends RefCounted
-class_name KiriPythonBuildWrangler
-
-var _python_release_info : Dictionary = {}
-
-func _get_python_release_info():
-	if _python_release_info == {}:
-		var this_script_path = get_script().resource_path
-		var this_script_dir = this_script_path.get_base_dir()
-		var release_info_path = this_script_dir.path_join("StandalonePythonBuilds/python_release_info.json")
-		_python_release_info = load(release_info_path).data
-	return _python_release_info
-
-func _get_python_version():
-	
-	var info = _get_python_release_info()
-	var versions : Array = info["versions"]
-	
-	# Sort version numbers so that the highest version is the first element.
-	versions.sort_custom(func(a : String, b : String):
-		var version_parts_a : PackedStringArray = a.split(".")
-		var version_parts_b : PackedStringArray = b.split(".")
-		for i in range(0, 3):
-			if int(version_parts_a[i]) > int(version_parts_b[i]):
-				return true
-			if int(version_parts_a[i]) < int(version_parts_b[i]):
-				return false
-		return false)
-
-	return versions[0]
-
-func _get_python_release():
-	var info = _get_python_release_info()
-	return info["release"]
-
-func _generate_python_archive_string(
-	python_version : String,
-	python_release : String,
-	arch : String,
-	os : String,
-	opt : String):
-
-	return "cpython-{python_version}+{python_release}-{python_arch}-{python_os}-{python_opt}-full.tar.zip".format(
-		{
-			"python_version" : python_version,
-			"python_release" : python_release,
-			"python_arch" : arch,
-			"python_os" : os,
-			"python_opt" : opt
-		})
-
-func _get_python_opt_for_os(os_name : String) -> String:
-	if os_name == "pc-windows-msvc-shared":
-		return "pgo"
-	return "pgo+lto"
-
-func _detect_archive_for_runtime( \
-	python_version : String,
-	python_release : String):
-
-	var arch : String = _get_python_architecture(Engine.get_architecture_name())
-	var os_name : String = _get_python_platform(OS.get_name(), arch)
-	var opt = _get_python_opt_for_os(os_name)
-	
-	var archive_str : String = _generate_python_archive_string(
-		python_version, python_release,
-		arch, os_name, opt)
-	
-	return archive_str
-
-func _detect_archive_for_build(
-	python_version : String,
-	python_release : String,
-	arch : String,
-	os_name : String):
-
-	var opt = _get_python_opt_for_os(os_name)
-	
-	var archive_str : String = _generate_python_archive_string(
-		python_version, python_release,
-		arch, os_name, opt)
-	
-	return archive_str
-
-# Note: arch variable is output of _get_python_architecture, not whatever Godot
-# returns.
-func _get_python_platform(os_name : String, arch : String) -> String:
-	var os_name_mappings : Dictionary = {
-		"Linux" : "unknown-linux-gnu",
-		"macOS" : "apple-darwin",
-		"Windows" : "pc-windows-msvc-shared"
-	}
-	
-	# Special case for armv7 Linux:
-	if arch == "armv7" and os_name == "Linux":
-		return "linux-gnueabi"
-	
-	assert(os_name_mappings.has(os_name))
-	return os_name_mappings[os_name]
-
-func _get_python_architecture(engine_arch : String) -> String:
-	var arch_name_mappings : Dictionary = {
-		"x86_64" : "x86_64",
-		"x86_32" : "i686",
-		"arm64" : "aarch64", # FIXME: I dunno if this is correct.
-		"arm32" : "armv7",  # FIXME: I dunno if this is correct.
-	}
-	assert(arch_name_mappings.has(engine_arch))
-	return arch_name_mappings[engine_arch]
-
-func _get_cache_path_relative():
-	return "_python_dist".path_join(_get_python_release()).path_join(_get_python_version())
-
-func _get_cache_path_system():
-	return OS.get_user_data_dir().path_join(_get_cache_path_relative())
-
-func _get_cache_path_godot():
-	return "user://".path_join(_get_cache_path_relative())
-
-func _get_runtime_python_archive_godot_path() -> String:
-	var this_script_path = get_script().resource_path
-	var this_script_dir = this_script_path.get_base_dir()
-	var python_archive_path = this_script_dir.path_join(
-		"StandalonePythonBuilds").path_join(
-		_detect_archive_for_runtime(
-			_get_python_version(),
-			_get_python_release()))
-	
-	return python_archive_path
-
-func get_export_python_archive_godot_path(platform : String, arch : String) -> String:
-	var this_script_path = get_script().resource_path
-	var this_script_dir = this_script_path.get_base_dir()
-	var python_archive_path = this_script_dir.path_join(
-		"StandalonePythonBuilds").path_join(
-		_detect_archive_for_build(
-			_get_python_version(),
-			_get_python_release(), arch, platform))
-	
-	return python_archive_path
-
-func _unpack_python():
-	
-	var python_archive_path = _get_runtime_python_archive_godot_path()
-	var reader : TARReader = TARReader.new()
-	reader.open(python_archive_path)
-	
-	#print(reader.get_files())
-	
-	var file_list : PackedStringArray = reader.get_files()
-	
-	for relative_filename : String in file_list:
-		reader.unpack_file(_get_cache_path_godot(), relative_filename)
-	
-	pass
-
-func _get_runtime_python_executable_godot_path():
-	var base_dir = _get_cache_path_godot().path_join("python/install")
-	if OS.get_name() == "Windows":
-		return base_dir.path_join("python.exe")
-	else:
-		return base_dir.path_join("bin/python3")
-	
-	# TODO: Other platforms.
-	
-	
-
-# Testing code follows...
-
-#func _ready():
-	#
-	#print(_detect_archive_for_runtime(
-		#_get_python_version(), _get_python_release()))
-	#
-	#print(_get_cache_path_godot())
-	#print(_get_cache_path_system())
-	#print(_get_runtime_python_archive_godot_path())
-	#
-	##_unpack_python()
-	#print(_get_runtime_python_executable_godot_path())
-	#print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()))
-	#
-	#var out = []
-	#OS.execute(
-		#ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()),
-		#["asdfjknsdcjknsdcjknsdjkc"], out, true)
-	#print(out)
-	#
-#
-