From 5d3b74d7983d0a6ed2c28328abbfc22584449c28 Mon Sep 17 00:00:00 2001 From: Kiri Date: Sun, 14 Jul 2024 08:54:08 -0700 Subject: [PATCH] Work from last night. --- TarTest.tscn | 6 + TestPythonInExport.gd | 21 ++ .../KiriPythonRPCWrapper.gd | 14 +- .../PythonBuildExportPlugin.gd | 32 ++ .../PythonBuildWrangler.gd | 190 +++++++++++ .../StandalonePythonBuilds/README_python.md | 69 ++++ .../StandalonePythonBuilds/convert_zsts.bsh | 21 ++ .../StandalonePythonBuilds/update.bsh | 75 +++++ addons/kiripythonrpcwrapper/TARReader.gd | 302 ++++++++++++++++++ export_presets.cfg | 102 ++++++ project.godot | 1 + 11 files changed, 828 insertions(+), 5 deletions(-) create mode 100644 TarTest.tscn create mode 100644 TestPythonInExport.gd create mode 100644 addons/kiripythonrpcwrapper/PythonBuildExportPlugin.gd create mode 100644 addons/kiripythonrpcwrapper/PythonBuildWrangler.gd create mode 100644 addons/kiripythonrpcwrapper/StandalonePythonBuilds/README_python.md create mode 100755 addons/kiripythonrpcwrapper/StandalonePythonBuilds/convert_zsts.bsh create mode 100755 addons/kiripythonrpcwrapper/StandalonePythonBuilds/update.bsh create mode 100644 addons/kiripythonrpcwrapper/TARReader.gd create mode 100644 export_presets.cfg diff --git a/TarTest.tscn b/TarTest.tscn new file mode 100644 index 0000000..488fa0e --- /dev/null +++ b/TarTest.tscn @@ -0,0 +1,6 @@ +[gd_scene load_steps=2 format=3 uid="uid://dfgfueotq2kt6"] + +[ext_resource type="Script" path="res://TestPythonInExport.gd" id="1_o4sdc"] + +[node name="TarTest" type="Node"] +script = ExtResource("1_o4sdc") diff --git a/TestPythonInExport.gd b/TestPythonInExport.gd new file mode 100644 index 0000000..49465ef --- /dev/null +++ b/TestPythonInExport.gd @@ -0,0 +1,21 @@ +extends Node + +func _ready(): + var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new() + + bw._unpack_python() + + ##_unpack_python() + #print(_get_runtime_python_executable_godot_path()) + #print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path())) + # + var out = [] + var ret = OS.execute( + ProjectSettings.globalize_path(bw._get_runtime_python_executable_godot_path()), + ["--version"], out, true) + + print("Ret: ", ret) + print("Out: ", out) + + pass + diff --git a/addons/kiripythonrpcwrapper/KiriPythonRPCWrapper.gd b/addons/kiripythonrpcwrapper/KiriPythonRPCWrapper.gd index 8f7739c..874b290 100644 --- a/addons/kiripythonrpcwrapper/KiriPythonRPCWrapper.gd +++ b/addons/kiripythonrpcwrapper/KiriPythonRPCWrapper.gd @@ -1,11 +1,15 @@ @tool extends EditorPlugin -func _enter_tree(): - # Initialization of the plugin goes here. - pass +var python_build_export_plugin = null +func _enter_tree(): + assert(not python_build_export_plugin) + python_build_export_plugin = KiriPythonBuildExportPlugin.new() + add_export_plugin(python_build_export_plugin) func _exit_tree(): - # Clean-up of the plugin goes here. - pass + assert(python_build_export_plugin) + remove_export_plugin(python_build_export_plugin) + python_build_export_plugin = null + diff --git a/addons/kiripythonrpcwrapper/PythonBuildExportPlugin.gd b/addons/kiripythonrpcwrapper/PythonBuildExportPlugin.gd new file mode 100644 index 0000000..2086952 --- /dev/null +++ b/addons/kiripythonrpcwrapper/PythonBuildExportPlugin.gd @@ -0,0 +1,32 @@ +@tool +extends EditorExportPlugin +class_name KiriPythonBuildExportPlugin + +func _export_begin( + features : PackedStringArray, is_debug : bool, + path : String, flags : int): + + print("features: ", features) + + var build_wrangler : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new() + + var platform_list = [] + var arch_list = [] + + if "linux" in features: + #platform_list.append(build_wrangler._get_python_platform("Linux")) + platform_list.append("Linux") + if "windows" in features: + platform_list.append("Windows") + if "x86_64" in features: + arch_list.append("x86_64") + + for platform in platform_list: + for arch in arch_list: + var python_arch : String = build_wrangler._get_python_architecture(arch) + var python_platform : String = build_wrangler._get_python_platform(platform, arch) + var archive_to_export = build_wrangler.get_export_python_archive_godot_path(python_platform, python_arch) + #print("EXPORT ME TOO: ", archive_to_export) + var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export) + print("Adding file: ", archive_to_export, " ", len(file_contents)) + add_file(archive_to_export, file_contents, false) diff --git a/addons/kiripythonrpcwrapper/PythonBuildWrangler.gd b/addons/kiripythonrpcwrapper/PythonBuildWrangler.gd new file mode 100644 index 0000000..365715c --- /dev/null +++ b/addons/kiripythonrpcwrapper/PythonBuildWrangler.gd @@ -0,0 +1,190 @@ +extends RefCounted +class_name KiriPythonBuildWrangler + +var _python_release_info : Dictionary = {} + +func _get_python_release_info(): + if _python_release_info == {}: + var this_script_path = get_script().resource_path + var this_script_dir = this_script_path.get_base_dir() + var release_info_path = this_script_dir.path_join("StandalonePythonBuilds/python_release_info.json") + _python_release_info = load(release_info_path).data + return _python_release_info + +func _get_python_version(): + + var info = _get_python_release_info() + var versions : Array = info["versions"] + + # Sort version numbers so that the highest version is the first element. + versions.sort_custom(func(a : String, b : String): + var version_parts_a : PackedStringArray = a.split(".") + var version_parts_b : PackedStringArray = b.split(".") + for i in range(0, 3): + if int(version_parts_a[i]) > int(version_parts_b[i]): + return true + if int(version_parts_a[i]) < int(version_parts_b[i]): + return false + return false) + + return versions[0] + +func _get_python_release(): + var info = _get_python_release_info() + return info["release"] + +func _generate_python_archive_string( + python_version : String, + python_release : String, + arch : String, + os : String, + opt : String): + + return "cpython-{python_version}+{python_release}-{python_arch}-{python_os}-{python_opt}-full.tar.zip".format( + { + "python_version" : python_version, + "python_release" : python_release, + "python_arch" : arch, + "python_os" : os, + "python_opt" : opt + }) + +func _get_python_opt_for_os(os_name : String) -> String: + if os_name == "pc-windows-msvc-shared": + return "pgo" + return "pgo+lto" + +func _detect_archive_for_runtime( \ + python_version : String, + python_release : String): + + var arch : String = _get_python_architecture(Engine.get_architecture_name()) + var os_name : String = _get_python_platform(OS.get_name(), arch) + var opt = _get_python_opt_for_os(os_name) + + var archive_str : String = _generate_python_archive_string( + python_version, python_release, + arch, os_name, opt) + + return archive_str + +func _detect_archive_for_build( + python_version : String, + python_release : String, + arch : String, + os_name : String): + + var opt = _get_python_opt_for_os(os_name) + + var archive_str : String = _generate_python_archive_string( + python_version, python_release, + arch, os_name, opt) + + return archive_str + +# Note: arch variable is output of _get_python_architecture, not whatever Godot +# returns. +func _get_python_platform(os_name : String, arch : String) -> String: + var os_name_mappings : Dictionary = { + "Linux" : "unknown-linux-gnu", + "macOS" : "apple-darwin", + "Windows" : "pc-windows-msvc-shared" + } + + # Special case for armv7 Linux: + if arch == "armv7" and os_name == "Linux": + return "linux-gnueabi" + + assert(os_name_mappings.has(os_name)) + return os_name_mappings[os_name] + +func _get_python_architecture(engine_arch : String) -> String: + var arch_name_mappings : Dictionary = { + "x86_64" : "x86_64", + "x86_32" : "i686", + "arm64" : "aarch64", # FIXME: I dunno if this is correct. + "arm32" : "armv7", # FIXME: I dunno if this is correct. + } + assert(arch_name_mappings.has(engine_arch)) + return arch_name_mappings[engine_arch] + +func _get_cache_path_relative(): + return "_python_dist".path_join(_get_python_release()).path_join(_get_python_version()) + +func _get_cache_path_system(): + return OS.get_user_data_dir().path_join(_get_cache_path_relative()) + +func _get_cache_path_godot(): + return "user://".path_join(_get_cache_path_relative()) + +func _get_runtime_python_archive_godot_path() -> String: + var this_script_path = get_script().resource_path + var this_script_dir = this_script_path.get_base_dir() + var python_archive_path = this_script_dir.path_join( + "StandalonePythonBuilds").path_join( + _detect_archive_for_runtime( + _get_python_version(), + _get_python_release())) + + return python_archive_path + +func get_export_python_archive_godot_path(platform : String, arch : String) -> String: + var this_script_path = get_script().resource_path + var this_script_dir = this_script_path.get_base_dir() + var python_archive_path = this_script_dir.path_join( + "StandalonePythonBuilds").path_join( + _detect_archive_for_build( + _get_python_version(), + _get_python_release(), arch, platform)) + + return python_archive_path + +func _unpack_python(): + + var python_archive_path = _get_runtime_python_archive_godot_path() + var reader : TARReader = TARReader.new() + reader.open(python_archive_path) + + #print(reader.get_files()) + + var file_list : PackedStringArray = reader.get_files() + + for relative_filename : String in file_list: + reader.unpack_file(_get_cache_path_godot(), relative_filename) + + pass + +func _get_runtime_python_executable_godot_path(): + var base_dir = _get_cache_path_godot().path_join("python/install") + if OS.get_name() == "Windows": + return base_dir.path_join("python.exe") + else: + return base_dir.path_join("bin/python3") + + # TODO: Other platforms. + + + +# Testing code follows... + +#func _ready(): + # + #print(_detect_archive_for_runtime( + #_get_python_version(), _get_python_release())) + # + #print(_get_cache_path_godot()) + #print(_get_cache_path_system()) + #print(_get_runtime_python_archive_godot_path()) + # + ##_unpack_python() + #print(_get_runtime_python_executable_godot_path()) + #print(ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path())) + # + #var out = [] + #OS.execute( + #ProjectSettings.globalize_path(_get_runtime_python_executable_godot_path()), + #["asdfjknsdcjknsdcjknsdjkc"], out, true) + #print(out) + # +# + diff --git a/addons/kiripythonrpcwrapper/StandalonePythonBuilds/README_python.md b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/README_python.md new file mode 100644 index 0000000..915b3b2 --- /dev/null +++ b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/README_python.md @@ -0,0 +1,69 @@ +# What + +This is where we store standalone Python builds for distributing with the built application. These will get unpacked into a temp directory on desktop platforms so that we can have an extremely specific, isolated Python environment. + +Standalone Python builds from: + + https://github.com/indygreg/python-build-standalone/releases + +# .tar.zst vs .tar.zip vs .tar.bz2 vs tar.gz, etc + +We're going to unpack these at runtime so they need to exist in a way that Godot can load. It's fairly simple for us to write our own .tar format parser, but for the compression format (zst, zip, gz, bz2, etc) it's better to rely on the engine's built-in decompression code. + +Of these formats, the only one that can be read by Godot (without Godot-specific headers being attached by saving the file from Godot) is the .zip format. Unfortunately, .zip format doesn't include a lot of the file permissions that the original .tar.whatever archive includes. + +So we're splitting the difference in an slightly unusual way: Use .zip as the compression around the .tar file instead of bzip2, gzip, zstd, it whatever else, then write a .tar parser to load the internal .tar at runtime. What we get from this is a slightly worse compression format that Godot can actually read at runtime, which preserves file permissions and other attributes the way a .tar would. + +For format reference on .tar: + + https://www.gnu.org/software/tar/manual/html_node/Standard.html + https://www.ibm.com/docs/en/zos/2.4.0?topic=formats-tar-format-tar-archives + +# The Process + +## Automated way + +Run `update.bsh`. + +## Obsolete, manual way + +### 1. Grab latest archives + +To update the archives here, grab the latest archive from: + + https://github.com/indygreg/python-build-standalone/releases + +There's a huge list of files there, so here's how to determine the latest version for this project: + +cpython-PYTHONVERSION+RELEASE-ARCHITECTURE-PLATFORM-OPTIMIZATIONS.tar.zst + + - PYTHONVERSION: The Python version. Unless there's a good reason to, you probably want the latest version of this. + + - RELEASE: Should correspond to the latest release. Formatted as a date (YYYYMMDD). + + - ARCHITECTURE: CPU architecture. This is going to be funky for Macs, but for desktop Linux/Windows PCs we usually just want `x86_64`. `x86_64_v2` and up include instructions found in newer and newer architectures. This may change if we start supporting stuff like Linux on ARM or whatever. + + - PLATFORM: + - For Windows we want `windows-msvc-shared`. + - For Linux we want `unknown-linux-gnu`. + + - OPTIMIZATIONS: `pgo+lto` for fully optimized builds. + +Examples: + + - Linux Python 3.12.3, release 20240415: `cpython-3.12.3+20240415-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst` + + - Windows Python 3.12.3, release 20240415: `cpython-3.12.3+20240415-x86_64-pc-windows-msvc-shared-pgo-full.tar.zst` + +See here for more info: + + https://gregoryszorc.com/docs/python-build-standalone/20240415/running.html + +### 2. Stick them in this directory + +### 3. Run the conversion script + +Run `./convert_zsts.bsh` in this directory. + +### 4. Add them to git + diff --git a/addons/kiripythonrpcwrapper/StandalonePythonBuilds/convert_zsts.bsh b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/convert_zsts.bsh new file mode 100755 index 0000000..4bc656c --- /dev/null +++ b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/convert_zsts.bsh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Why do we do this? Because Godot can read zip files but not zst +# files. But we still want to preserve the file attributes in the .tar +# archive. + + +cd "$(dirname "$0")" + +# Decompress zsts... +for ZSTFILE in *.tar.zst; do + if [ \! -e "$(basename ${ZSTFILE} .zst)" ]; then + zstd -d "${ZSTFILE}" + fi +done + +# Recompress zips... +for TARFILE in *.tar; do + zip "${TARFILE}.zip" "${TARFILE}" +done + diff --git a/addons/kiripythonrpcwrapper/StandalonePythonBuilds/update.bsh b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/update.bsh new file mode 100755 index 0000000..1ca2efa --- /dev/null +++ b/addons/kiripythonrpcwrapper/StandalonePythonBuilds/update.bsh @@ -0,0 +1,75 @@ +#!/bin/bash + +# I am writing this with an ocular migraine and it's hard as shit to +# real the goddamn code, so please excuse any obvious mistakes. I +# literally cannot see them right now. + +PYTHON_VERSIONS="3.12.3" + +# TODO: Add more to this list if we want to support more platforms. +PYTHON_PLATFORM_CONFIGS="x86_64-pc-windows-msvc-shared-pgo-full x86_64-unknown-linux-gnu-pgo+lto-full" + +set -e + +cd "$(dirname "$0")" + +# wget \ +# https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json \ +# -o latest-release.json + +RELEASE_PARTS=$(curl https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json | \ + python3 -c "import json; import sys; d = json.loads(sys.stdin.read()); print(d[\"tag\"]); print(d[\"asset_url_prefix\"]);") + + +RELEASE_TAG="$(echo $RELEASE_PARTS | cut -d" " -f 1)" +RELEASE_BASE_URL="$(echo $RELEASE_PARTS | cut -d" " -f 2)" + +echo $RELEASE_TAG +echo $RELEASE_BASE_URL + +echo "Fetching new files from release..." + +for PYTHON_VERSION in $PYTHON_VERSIONS; do + + for CONFIG in $PYTHON_PLATFORM_CONFIGS; do + FILENAME="cpython-${PYTHON_VERSION}+${RELEASE_TAG}-$CONFIG.tar.zst" + if [ \! -e "${FILENAME}" ] ; then + wget "${RELEASE_BASE_URL}/${FILENAME}" + fi + done + +done + +echo "Decompressing zsts..." + +# Decompress zsts... +for ZSTFILE in *.tar.zst; do + if [ \! -e "$(basename ${ZSTFILE} .zst)" ]; then + zstd -d "${ZSTFILE}" + fi +done + +echo "Compressing zips..." + +# Recompress zips... +for TARFILE in *.tar; do + if [ \! -e "${TARFILE}.zip" ]; then + zip "${TARFILE}.zip" "${TARFILE}" + fi +done + +# Write version data. +# FIXME: Extremely dirty and hacky. +echo "{ \"release\" : \"${RELEASE_TAG}\", \"versions\" : [" > python_release_info.json +FIRST="1" +for PYTHON_VERSION in $PYTHON_VERSIONS; do + if [ "$FIRST" == "0" ]; then + echo "," >> python_release_info.json + fi + FIRST=0 + echo "\"${PYTHON_VERSION}\"" >> python_release_info.json +done +echo "]" >> python_release_info.json +echo "}" >> python_release_info.json + + diff --git a/addons/kiripythonrpcwrapper/TARReader.gd b/addons/kiripythonrpcwrapper/TARReader.gd new file mode 100644 index 0000000..d430770 --- /dev/null +++ b/addons/kiripythonrpcwrapper/TARReader.gd @@ -0,0 +1,302 @@ +# TARReader +# +# Read .tar.zip files. Interface mostly identical to ZIPReader. +# +# Why .tar.zip instead of .tar.bz2, .tar.gz, .tar.zst, or something normal? +# Godot supports loading files with GZip and Zstandard compression, but only +# files that it's saved (with a header/footer), so it can't load normal .tar.gz +# or .tar.zst files. It can load zips, though. +# +# DO NOT USE THIS ON UNTRUSTED DATA. + +extends RefCounted +class_name TARReader + +#region Internal data + +class TarFileRecord: + extends RefCounted + var filename : String + var offset : int + var file_size : int + + # Unix file permissions. + # + # Technically this is an int, but we're just going to leave it as an octal + # string because that's what we can feed right into chmod. + var mode : String + + # Symlinks. + var is_link : bool + var link_destination : String + + var is_directory : bool + + var type_indicator : String + +var _internal_file_list = [] +var _reader : ZIPReader = null +var _tar_file_cache : PackedByteArray = [] + +#endregion + +#region Cache wrangling + +# We have to load the entire .tar file into memory with the way the ZipReader +# API works, but we'll at least include an option to nuke the cache to free up +# memory if you want to just leave the file open. +# +# This lets us avoid re-opening and decompressing the entire .tar every time we +# need something out of it, while still letting us manually free the memory when +# we won't need it for a while. +func clear_cache(): + _tar_file_cache = [] + +func load_cache() -> Error: + assert(_reader) + + if len(_tar_file_cache): + # Cache already in-memory. + return OK + + var zip_file_list = _reader.get_files() + if len(zip_file_list) != 1: + return ERR_FILE_UNRECOGNIZED + + _tar_file_cache = _reader.read_file(zip_file_list[0]) + + return OK + +#endregion + +func close() -> Error: + _internal_file_list = [] + _reader.close() + _reader = null + clear_cache() + return OK + +func file_exists(path: String, case_sensitive: bool = true) -> bool: + for record : TarFileRecord in _internal_file_list: + if case_sensitive: + if record.filename == path: + return true + else: + if record.filename.nocasecmp_to(path) == 0: + return true + return false + +func get_files() -> PackedStringArray: + var ret : PackedStringArray = [] + for record : TarFileRecord in _internal_file_list: + ret.append(record.filename) + return ret + +func _octal_str_to_int(s : String) -> int: + var ret : int = 0; + var digit_multiplier = 1; + while len(s): + var lsb = s.substr(len(s) - 1, 1) + s = s.substr(0, len(s) - 1) + ret += digit_multiplier * lsb.to_int() + digit_multiplier *= 8 + return ret + +func _pad_to_512(x : int) -> int: + var x_lowbits = x & 511 + var x_hibits = x & ~511 + + if x_lowbits: + x_hibits += 512 + + return x_hibits + +func open(path: String) -> Error: + + assert(not _reader) + _reader = ZIPReader.new() + var err = _reader.open(path) + if err != OK: + _reader.close() + _reader = null + return err + + load_cache() + + var tar_file_offset = 0 + var zero_filled_record_count = 0 + var zero_filled_record : PackedByteArray = [] + zero_filled_record.resize(512) + zero_filled_record.fill(0) + + var paxheader_next_file = {} + var paxheader_global = {} + + while tar_file_offset < len(_tar_file_cache): + var chunk = _tar_file_cache.slice(tar_file_offset, tar_file_offset + 512) + + if chunk == zero_filled_record: + zero_filled_record_count += 1 + if zero_filled_record_count >= 2: + break + tar_file_offset += 512 + continue + + var tar_record : TarFileRecord = TarFileRecord.new() + + var tar_chunk_name = chunk.slice(0, 100) + var tar_chunk_size = chunk.slice(124, 124+12) + var tar_chunk_mode = chunk.slice(100, 100+8) + var tar_chunk_link_indicator = chunk.slice(156, 156+1) + var tar_chunk_link_file = chunk.slice(157, 157+100) + + # FIXME: Technically "ustar\0" but we'll skip the \0 + var tar_ustar_indicator = chunk.slice(257, 257+5) + var tar_ustar_file_prefix = chunk.slice(345, 345+155) + + # Pluck out the relevant bits we need for the record. + tar_record.filename = tar_chunk_name.get_string_from_utf8() + + tar_record.file_size = _octal_str_to_int(tar_chunk_size.get_string_from_utf8()) + tar_record.mode = tar_chunk_mode.get_string_from_utf8() + tar_record.is_link = (tar_chunk_link_indicator[0] != 0 and tar_chunk_link_indicator.get_string_from_utf8()[0] == "2") + tar_record.link_destination = tar_chunk_link_file.get_string_from_utf8() + + tar_record.is_directory = (tar_chunk_link_indicator[0] != 0 and tar_chunk_link_indicator.get_string_from_utf8()[0] == "5") + + if tar_chunk_link_indicator[0] != 0: + tar_record.type_indicator = tar_chunk_link_indicator.get_string_from_utf8() + else: + tar_record.type_indicator = "" + + # Append prefix if this is the "ustar" format. + # TODO: Test this. + if tar_ustar_indicator.get_string_from_utf8() == "ustar": + tar_record.filename = \ + tar_ustar_file_prefix.get_string_from_utf8() + \ + tar_record.filename + + # TODO: Things we skipped: + # - owner id (108, 108+8) + # - group id (116, 116+8) + # - modification time (136, 136+12) + # - checksum (148, 148+8) + # - mosty related to USTAR format + + # Skip header. + tar_file_offset += 512 + + # Record start offset. + tar_record.offset = tar_file_offset + + # Skip file contents. + tar_file_offset += _pad_to_512(tar_record.file_size) + + if tar_record.filename.get_file() == "@PaxHeader": + + # This is a special file entry that just has some extended data + # about the next file or all the following files. It's not an actual + # file. + var paxheader_data : PackedByteArray = _tar_file_cache.slice( + tar_record.offset, + tar_record.offset + tar_record.file_size) + + var paxheader_str : String = paxheader_data.get_string_from_utf8() + + # FIXME: Do some error checking here. + var paxheader_lines = paxheader_str.split("\n", false) + for line in paxheader_lines: + var length_and_the_rest = line.split(" ") + var key_and_value = length_and_the_rest[1].split("=") + var key = key_and_value[0] + var value = key_and_value[1] + + if tar_record.type_indicator == "x": + paxheader_next_file[key] = value + elif tar_record.type_indicator == "g": + paxheader_global[key] = value + + else: + + # Apply paxheader. We're just using "path" for now. + # See here for other available fields: + # https://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html + var merged_paxheader : Dictionary = paxheader_global.duplicate() + merged_paxheader.merge(paxheader_next_file, true) + paxheader_next_file = {} + + if merged_paxheader.has("path"): + tar_record.filename = merged_paxheader["path"] + print("fixing path for paxheader: ", tar_record.filename) + + # Add it to our record list. + _internal_file_list.append(tar_record) + + return OK + +func _load_record(record : TarFileRecord) -> PackedByteArray: + load_cache() + return _tar_file_cache.slice(record.offset, record.offset + record.file_size) + +func read_file(path : String, case_sensitive : bool = true) -> PackedByteArray: + + for record : TarFileRecord in _internal_file_list: + if case_sensitive: + if record.filename == path: + return _load_record(record) + else: + if record.filename.nocasecmp_to(path) == 0: + return _load_record(record) + + return [] + +func unpack_file(dest_path : String, filename : String): + var full_dest_path : String = dest_path.path_join(filename) + DirAccess.make_dir_recursive_absolute(full_dest_path.get_base_dir()) + + for record : TarFileRecord in _internal_file_list: + + if record.filename.is_absolute_path(): + # hmmmmmmmmmmmmmm + assert(false) + continue + + if record.filename.simplify_path().begins_with(".."): + assert(false) + continue + + # FIXME: There are probably a million other ways to do directory + # traversal attacks. + + if record.filename == filename: + if record.is_link: + + # Okay, look. I know that symbolic links technically exist on + # Windows, but they're messy and hardly ever used. FIXME later + # if for some reason you need to support that. -Kiri + assert(OS.get_name() != "Windows") + + var err = OS.execute("ln", [ + "-s", + record.link_destination, + ProjectSettings.globalize_path(full_dest_path) ]) + assert(err != -1) + + elif record.is_directory: + + DirAccess.make_dir_recursive_absolute(full_dest_path) + + else: + + var file_data : PackedByteArray = read_file(record.filename) + var out_file = FileAccess.open(full_dest_path, FileAccess.WRITE) + out_file.store_buffer(file_data) + out_file.close() + + # Set permissions. + if not record.is_link: + if OS.get_name() != "Windows": + var err = OS.execute("chmod", [ + record.mode, + ProjectSettings.globalize_path(full_dest_path) ]) + assert(err != -1) diff --git a/export_presets.cfg b/export_presets.cfg new file mode 100644 index 0000000..c7247e7 --- /dev/null +++ b/export_presets.cfg @@ -0,0 +1,102 @@ +[preset.0] + +name="Linux/X11" +platform="Linux/X11" +runnable=true +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="./GodotJSONRPCTest.x86_64" +encryption_include_filters="" +encryption_exclude_filters="" +encrypt_pck=false +encrypt_directory=false + +[preset.0.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +texture_format/bptc=true +texture_format/s3tc=true +texture_format/etc=false +texture_format/etc2=false +binary_format/architecture="x86_64" +ssh_remote_deploy/enabled=false +ssh_remote_deploy/host="user@host_ip" +ssh_remote_deploy/port="22" +ssh_remote_deploy/extra_args_ssh="" +ssh_remote_deploy/extra_args_scp="" +ssh_remote_deploy/run_script="#!/usr/bin/env bash +export DISPLAY=:0 +unzip -o -q \"{temp_dir}/{archive_name}\" -d \"{temp_dir}\" +\"{temp_dir}/{exe_name}\" {cmd_args}" +ssh_remote_deploy/cleanup_script="#!/usr/bin/env bash +kill $(pgrep -x -f \"{temp_dir}/{exe_name} {cmd_args}\") +rm -rf \"{temp_dir}\"" + +[preset.1] + +name="Windows Desktop" +platform="Windows Desktop" +runnable=true +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="./GodotJSONRPCTest.exe" +encryption_include_filters="" +encryption_exclude_filters="" +encrypt_pck=false +encrypt_directory=false + +[preset.1.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +texture_format/bptc=true +texture_format/s3tc=true +texture_format/etc=false +texture_format/etc2=false +binary_format/architecture="x86_64" +codesign/enable=false +codesign/timestamp=true +codesign/timestamp_server_url="" +codesign/digest_algorithm=1 +codesign/description="" +codesign/custom_options=PackedStringArray() +application/modify_resources=true +application/icon="" +application/console_wrapper_icon="" +application/icon_interpolation=4 +application/file_version="" +application/product_version="" +application/company_name="" +application/product_name="" +application/file_description="" +application/copyright="" +application/trademarks="" +application/export_angle=0 +ssh_remote_deploy/enabled=false +ssh_remote_deploy/host="user@host_ip" +ssh_remote_deploy/port="22" +ssh_remote_deploy/extra_args_ssh="" +ssh_remote_deploy/extra_args_scp="" +ssh_remote_deploy/run_script="Expand-Archive -LiteralPath '{temp_dir}\\{archive_name}' -DestinationPath '{temp_dir}' +$action = New-ScheduledTaskAction -Execute '{temp_dir}\\{exe_name}' -Argument '{cmd_args}' +$trigger = New-ScheduledTaskTrigger -Once -At 00:00 +$settings = New-ScheduledTaskSettingsSet +$task = New-ScheduledTask -Action $action -Trigger $trigger -Settings $settings +Register-ScheduledTask godot_remote_debug -InputObject $task -Force:$true +Start-ScheduledTask -TaskName godot_remote_debug +while (Get-ScheduledTask -TaskName godot_remote_debug | ? State -eq running) { Start-Sleep -Milliseconds 100 } +Unregister-ScheduledTask -TaskName godot_remote_debug -Confirm:$false -ErrorAction:SilentlyContinue" +ssh_remote_deploy/cleanup_script="Stop-ScheduledTask -TaskName godot_remote_debug -ErrorAction:SilentlyContinue +Unregister-ScheduledTask -TaskName godot_remote_debug -Confirm:$false -ErrorAction:SilentlyContinue +Remove-Item -Recurse -Force '{temp_dir}'" diff --git a/project.godot b/project.godot index 13c483a..1e42402 100644 --- a/project.godot +++ b/project.godot @@ -11,6 +11,7 @@ config_version=5 [application] config/name="GodotJSONRPCTest" +run/main_scene="res://TarTest.tscn" config/features=PackedStringArray("4.2", "GL Compatibility") run/max_fps=60 config/icon="res://icon.svg"