More stuff.

This commit is contained in:
Kiri 2024-07-14 12:58:20 -07:00
parent bdd56c30d1
commit 4cad6a574f
16 changed files with 388 additions and 217 deletions

View File

@ -1,3 +1,6 @@
# FIXME: Remove this. I think we can nuke this entire class because we don't
# need it anymore.
# KiriJSONRPC # KiriJSONRPC
# #
# This just wraps JSONRPC and adds a little more sanity-checking, like # This just wraps JSONRPC and adds a little more sanity-checking, like

View File

@ -1,17 +1,13 @@
extends Node extends Node
func _ready(): func _ready():
var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
bw.unpack_python()
var out = []
var ret = OS.execute(
bw.get_runtime_python_executable_system_path(),
["--version"], out, true)
print("Ret: ", ret) var pw : KiriPythonWrapperInstance = KiriPythonWrapperInstance.new(
print("Out: ", out) "/storage/git2/GodotJSONRPCTest/addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/test_module/__init__.py")
pass pw.setup_python()
pw.start_process()
var ret = pw.call_rpc_sync("func_to_call", ["test string whatever blah"])
print(ret)

View File

@ -1,13 +1,20 @@
# Python build export plugin
#
# This just makes sure that the specific Python build for whatever platform we
# need gets bundled into the build for that platform, so that it can be unpacked
# and used later by KiriPythonBuildWrangler.
@tool @tool
extends EditorExportPlugin extends EditorExportPlugin
class_name KiriPythonBuildExportPlugin class_name KiriPythonBuildExportPlugin
func _get_name() -> String:
return "KiriPythonBuildExportPlugin"
func _export_begin( func _export_begin(
features : PackedStringArray, is_debug : bool, features : PackedStringArray, is_debug : bool,
path : String, flags : int): path : String, flags : int):
print("features: ", features)
var build_wrangler : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new() var build_wrangler : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
var platform_list = [] var platform_list = []
@ -26,5 +33,26 @@ func _export_begin(
for arch in arch_list: for arch in arch_list:
var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch) var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch)
var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export) var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export)
print("Adding file: ", archive_to_export, " ", len(file_contents))
add_file(archive_to_export, file_contents, false) add_file(archive_to_export, file_contents, false)
# Make sure all the RPC wrapper scripts make it in.
var script_path : String = get_script().resource_path
var script_dir : String = script_path.get_base_dir()
# Actually add all the files.
var extra_python_files = build_wrangler.get_extra_scripts_list()
for extra_python_file : String in extra_python_files:
var file_bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_python_file)
add_file(extra_python_file, file_bytes, false)
# Add the list of Python files as its own file so we know what to extract so
# it's visible to Python.
var python_wrapper_manifest_str : String = JSON.stringify(extra_python_files, " ")
var python_wrapper_manifest_bytes : PackedByteArray = \
python_wrapper_manifest_str.to_utf8_buffer()
var python_wrapper_manifset_path = script_dir.path_join(
"KiriPythonWrapperPythonFiles.json")
add_file(python_wrapper_manifset_path, python_wrapper_manifest_bytes, false)

View File

@ -1,3 +1,7 @@
# Python build wrangler
#
# This handles extracting and juggling standalone Python builds per-platform.
extends RefCounted extends RefCounted
class_name KiriPythonBuildWrangler class_name KiriPythonBuildWrangler
@ -217,7 +221,7 @@ func unpack_python(overwrite : bool = false):
# Open archive. # Open archive.
var python_archive_path : String = _detect_archive_for_runtime() var python_archive_path : String = _detect_archive_for_runtime()
var reader : TARReader = TARReader.new() var reader : KiriTARReader = KiriTARReader.new()
var err : Error = reader.open(python_archive_path) var err : Error = reader.open(python_archive_path)
assert(err == OK) assert(err == OK)
@ -230,4 +234,66 @@ func unpack_python(overwrite : bool = false):
# TODO: Clear cache function. Uninstall Python, etc. # TODO: Clear cache function. Uninstall Python, etc.
func get_extra_scripts_list() -> Array:
var script_path : String = get_script().resource_path
var script_dir : String = script_path.get_base_dir()
var python_wrapper_manifset_path = script_dir.path_join(
"KiriPythonWrapperPythonFiles.json")
# If this is running an actual build, we'll just return the manifest here.
if FileAccess.file_exists(python_wrapper_manifset_path):
return load(python_wrapper_manifset_path).data
# If it's not running an actual build, we need to scan for extra Python
# files.
# First pass: Find all the .kiri_export_python markers in the entire project
# tree.
var extra_python_files : Array = []
var scan_dir_list = ["res://"]
var verified_script_bundles = []
while len(scan_dir_list):
var current_dir : String = scan_dir_list.pop_front()
var da : DirAccess = DirAccess.open(current_dir)
if da.file_exists(".kiri_export_python"):
verified_script_bundles.append(current_dir)
else:
# Add all directories to the scan list.
da.include_navigational = false
var dir_list = da.get_directories()
for dir in dir_list:
if dir == "__pycache__":
continue
scan_dir_list.append(current_dir.path_join(dir))
# Second pass: Add everything under a directory containing a
# .kiri_export_python marker.
scan_dir_list = verified_script_bundles
while len(scan_dir_list):
var current_dir : String = scan_dir_list.pop_front()
var da : DirAccess = DirAccess.open(current_dir)
# Add all directories to the scan list.
da.include_navigational = false
var dir_list = da.get_directories()
for dir in dir_list:
if dir == "__pycache__":
continue
scan_dir_list.append(current_dir.path_join(dir))
# Add all Python files.
var file_list = da.get_files()
for file in file_list:
var full_file = current_dir.path_join(file)
extra_python_files.append(full_file)
## FIXME: Remove this.
#for f in extra_python_files:
#print("Extra file: ", f)
return extra_python_files
#endregion #endregion

View File

@ -1,3 +1,10 @@
# KiriPacketSocket
#
# GDScript version of the KiriPacketSocket Python module. Basically just copied
# the code over and reformatted it. Error handling and some other behaviors are
# different due to differences in how Python and GDScript handle exceptions and
# others.
extends RefCounted extends RefCounted
class_name KiriPacketSocket class_name KiriPacketSocket

View File

@ -4,149 +4,158 @@ import importlib.util
import sys import sys
import argparse import argparse
import time import time
import psutil # import psutil
import json import json
import KiriPacketSocket import KiriPacketSocket
# Parse arguments # This whole thing being in a try/except is just so we can catch
arg_parser = argparse.ArgumentParser( # errors and see them before the terminal window closes.
prog="KiriPythonRPCWrapper", # try:
description="Wrapper for Python modules to RPCs from Godot.", if True:
epilog="")
arg_parser.add_argument("--script", type=str, required=True) # Parse arguments
arg_parser.add_argument("--port", type=int, required=True) arg_parser = argparse.ArgumentParser(
arg_parser.add_argument("--parent_pid", type=int, required=True) prog="KiriPythonRPCWrapper",
description="Wrapper for Python modules to RPCs from Godot.",
epilog="")
args = arg_parser.parse_args() arg_parser.add_argument("--script", type=str, required=True)
arg_parser.add_argument("--port", type=int, required=True)
arg_parser.add_argument("--parent_pid", type=int, required=True)
args = arg_parser.parse_args()
# module_path = "../KiriPacketSocket/__init__.py" # module_path = "../KiriPacketSocket/__init__.py"
# module_name = "KiriPacketSocket" # module_name = "KiriPacketSocket"
module_path = args.script module_path = args.script
module_name = "" module_name = ""
# Attempt to load the module. # Attempt to load the module.
module_spec = importlib.util.spec_from_file_location( module_spec = importlib.util.spec_from_file_location(
module_name, module_path) module_name, module_path)
module = importlib.util.module_from_spec(module_spec) module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module) module_spec.loader.exec_module(module)
# This will be all the functions we find in the module that don't # This will be all the functions we find in the module that don't
# start with "_". # start with "_".
known_entrypoints = {} known_entrypoints = {}
# Scan the module for "public" functions. # Scan the module for "public" functions.
for entrypoint in dir(module): for entrypoint in dir(module):
# Skip anything starting with "_". Probably not meant to be # Skip anything starting with "_". Probably not meant to be
# exposed. # exposed.
if entrypoint.startswith("_"): if entrypoint.startswith("_"):
continue continue
attr = getattr(module, entrypoint) attr = getattr(module, entrypoint)
# if hasattr(attr, "__call__"): # if hasattr(attr, "__call__"):
if callable(attr): if callable(attr):
known_entrypoints[entrypoint] = attr known_entrypoints[entrypoint] = attr
# Connect to server. # Connect to server.
packet_socket = KiriPacketSocket.PacketSocket() packet_socket = KiriPacketSocket.PacketSocket()
packet_socket.start_client(("127.0.0.1", args.port)) packet_socket.start_client(("127.0.0.1", args.port))
while packet_socket.get_state() == packet_socket.SocketState.CONNECTING: while packet_socket.get_state() == packet_socket.SocketState.CONNECTING:
time.sleep(0.001) time.sleep(0.001)
if packet_socket.get_state() != packet_socket.SocketState.CONNECTED: if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
packet_socket.stop() packet_socket.stop()
raise Exception("Failed to connect to RPC host.") raise Exception("Failed to connect to RPC host.")
print("Starting packet processing.") print("Starting packet processing.")
def send_error_response(code, message, request_id): def send_error_response(code, message, request_id):
ret_dict = {
"jsonrpc" : "2.0",
"error" : {
"code" : code,
"message" : message
},
"id" : request_id
}
ret_dict_json = json.dumps(ret_dict)
packet_socket.send_packet(ret_dict_json.encode("utf-8"))
def send_response(result, request_id):
try:
ret_dict = { ret_dict = {
"jsonrpc" : "2.0", "jsonrpc" : "2.0",
"result" : ret, "error" : {
"code" : code,
"message" : message
},
"id" : request_id "id" : request_id
} }
ret_dict_json = json.dumps(ret_dict) ret_dict_json = json.dumps(ret_dict)
packet_socket.send_packet(ret_dict_json.encode("utf-8")) packet_socket.send_packet(ret_dict_json.encode("utf-8"))
except Exception as e:
send_error_response(-32603, "Error sending result: " + str(e), request_id)
# Start processing packets. def send_response(result, request_id):
while True: try:
ret_dict = {
"jsonrpc" : "2.0",
"result" : ret,
"id" : request_id
}
ret_dict_json = json.dumps(ret_dict)
packet_socket.send_packet(ret_dict_json.encode("utf-8"))
except Exception as e:
send_error_response(-32603, "Error sending result: " + str(e), request_id)
# Shutdown when we lose connection to host. # Start processing packets.
if packet_socket.get_state() != packet_socket.SocketState.CONNECTED: while True:
packet_socket.stop()
raise Exception("Disconnected from RPC host.")
# Watch parent PID so we can clean up when needed. # Shutdown when we lose connection to host.
if not psutil.pid_exists(args.parent_pid): if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
packet_socket.stop() packet_socket.stop()
raise Exception("RPC host process died") raise Exception("Disconnected from RPC host.")
# # Watch parent PID so we can clean up when needed.
# if not psutil.pid_exists(args.parent_pid):
# packet_socket.stop()
# raise Exception("RPC host process died")
next_packet = packet_socket.get_next_packet()
while next_packet:
this_packet = next_packet
next_packet = packet_socket.get_next_packet() next_packet = packet_socket.get_next_packet()
while next_packet:
this_packet = next_packet
next_packet = packet_socket.get_next_packet()
print("GOT PACKET: ", this_packet) print("GOT PACKET: ", this_packet)
# FIXME: Handle batches. # FIXME: Handle batches.
# Parse the incoming dict. # Parse the incoming dict.
try: try:
request_dict_json = this_packet.decode("utf-8") request_dict_json = this_packet.decode("utf-8")
request_dict = json.loads(request_dict_json) request_dict = json.loads(request_dict_json)
except Exception as e: except Exception as e:
send_error_response(-32700, "Error parsing packet: " + str(e), request_id) send_error_response(-32700, "Error parsing packet: " + str(e), request_id)
continue continue
# Make sure all the fields are there. # Make sure all the fields are there.
try: try:
method = request_dict["method"] method = request_dict["method"]
func_args = request_dict["params"] func_args = request_dict["params"]
request_id = request_dict["id"] request_id = request_dict["id"]
except Exception as e: except Exception as e:
send_error_response(-32602, "Missing field: " + str(e), request_id) send_error_response(-32602, "Missing field: " + str(e), request_id)
continue continue
# Make sure the method is something we scanned earlier. # Make sure the method is something we scanned earlier.
try: try:
func = known_entrypoints[method] func = known_entrypoints[method]
except Exception as e: except Exception as e:
send_error_response(-32601, "Method not found: " + str(e), request_id) send_error_response(-32601, "Method not found: " + str(e), request_id)
continue continue
# Call the dang function. # Call the dang function.
try: try:
ret = func(*func_args) ret = func(*func_args)
except Exception as e: except Exception as e:
send_error_response(-32603, "Call failed: " + str(e), request_id) send_error_response(-32603, "Call failed: " + str(e), request_id)
continue continue
send_response(ret, request_id) send_response(ret, request_id)
time.sleep(0.0001)
time.sleep(0.0001)
# except Exception as e:
# sys.stderr.write(e)
# time.sleep(5)
# raise e

View File

@ -1,103 +1,11 @@
extends RefCounted extends RefCounted
class_name KiriPythonWrapperInstance class_name KiriPythonWrapperInstance
var external_process_pid = -1
var server_packet_socket : KiriPacketSocket = null
var communication_packet_socket : KiriPacketSocket = null
var python_script_path : String = ""
enum KiriPythonWrapperStatus { enum KiriPythonWrapperStatus {
STATUS_RUNNING, STATUS_RUNNING,
STATUS_STOPPED STATUS_STOPPED
} }
func _init(python_file_path : String):
python_script_path = python_file_path
func _get_python_executable():
# FIXME: Adjust per-OS. Maybe test a few locations.
return "/usr/bin/python3"
func _get_wrapper_script():
# FIXME: Paths will be different for builds.
var script_path = self.get_script().get_path()
var script_dirname = script_path.get_base_dir()
return ProjectSettings.globalize_path( \
script_dirname + "/KiriPythonRPCWrapper_start.py")
func get_status():
if external_process_pid == -1:
return KiriPythonWrapperStatus.STATUS_STOPPED
if not OS.is_process_running(external_process_pid):
return KiriPythonWrapperStatus.STATUS_STOPPED
return KiriPythonWrapperStatus.STATUS_RUNNING
func start_process():
# FIXME: Make sure we don't have one running.
var open_port = 9500
assert(not server_packet_socket)
server_packet_socket = KiriPacketSocket.new()
while true:
server_packet_socket.start_server(["127.0.0.1", open_port])
# Wait for the server to start.
while server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
OS.delay_usec(1)
# If we're successfully listening, then we found a port to use and we
# don't need to loop anymore.
if server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
break
# This port is busy. Try the next one.
server_packet_socket.stop()
open_port += 1
print("Port: ", open_port)
var python_exe_path : String = _get_python_executable()
var wrapper_script_path : String = _get_wrapper_script()
var startup_command : Array = [
"xterm", "-e",
python_exe_path,
wrapper_script_path,
"--script", python_script_path,
"--port", open_port,
"--parent_pid", OS.get_process_id()]
print("startup command: ", startup_command)
external_process_pid = OS.create_process(
startup_command[0], startup_command.slice(1), true)
print("external process: ", external_process_pid)
func stop_process():
if external_process_pid != -1:
OS.kill(external_process_pid)
external_process_pid = -1
# Clean up server and communication sockets.
if server_packet_socket:
server_packet_socket.stop()
server_packet_socket = null
if communication_packet_socket:
communication_packet_socket.stop()
communication_packet_socket = null
class KiriPythonWrapperActiveRequest: class KiriPythonWrapperActiveRequest:
enum KiriPythonWrapperActiveRequestState { enum KiriPythonWrapperActiveRequestState {
@ -117,6 +25,131 @@ class KiriPythonWrapperActiveRequest:
var _active_request_queue = {} var _active_request_queue = {}
var _request_counter = 0 var _request_counter = 0
var _server_packet_socket : KiriPacketSocket = null
var communication_packet_socket : KiriPacketSocket = null
var python_script_path : String = ""
var _build_wrangler : KiriPythonBuildWrangler = null
var _external_process_pid = -1
func _init(python_file_path : String):
_build_wrangler = KiriPythonBuildWrangler.new()
python_script_path = python_file_path
func _get_python_executable():
return _build_wrangler.get_runtime_python_executable_system_path()
func _get_wrapper_script():
# FIXME: Paths will be different for builds.
var script_path = self.get_script().get_path()
var script_dirname = script_path.get_base_dir()
return ProjectSettings.globalize_path( \
script_dirname + "/KiriPythonRPCWrapper_start.py")
func _get_wrapper_cache_path() -> String:
return _build_wrangler._get_cache_path_godot().path_join("KiriPythonRPCWrapper")
func _get_wrapper_script_cache_path() -> String:
return _get_wrapper_cache_path().path_join("addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/__init__.py")
func setup_python():
# Unpack base Python build.
_build_wrangler.unpack_python(false)
# Unpack Python wrapper.
var extra_scripts = _build_wrangler.get_extra_scripts_list()
print(extra_scripts)
for extra_script : String in extra_scripts:
# Chop off the "res://".
var extra_script_relative : String = extra_script.substr(len("res://"))
# Some other path wrangling.
var extraction_path : String = _get_wrapper_cache_path().path_join(extra_script_relative)
var extraction_path_dir : String = extraction_path.get_base_dir()
# Make the dir.
DirAccess.make_dir_recursive_absolute(extraction_path_dir)
# Extract the file.
var bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_script)
FileAccess.open(extraction_path, FileAccess.WRITE).store_buffer(bytes)
func get_status():
if _external_process_pid == -1:
return KiriPythonWrapperStatus.STATUS_STOPPED
if not OS.is_process_running(_external_process_pid):
return KiriPythonWrapperStatus.STATUS_STOPPED
return KiriPythonWrapperStatus.STATUS_RUNNING
func start_process():
# FIXME: Make sure we don't have one running.
var open_port = 9500
assert(not _server_packet_socket)
_server_packet_socket = KiriPacketSocket.new()
while true:
_server_packet_socket.start_server(["127.0.0.1", open_port])
# Wait for the server to start.
while _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
OS.delay_usec(1)
# If we're successfully listening, then we found a port to use and we
# don't need to loop anymore.
if _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
break
# This port is busy. Try the next one.
_server_packet_socket.stop()
open_port += 1
print("Port: ", open_port)
var python_exe_path : String = _get_python_executable()
var wrapper_script_path : String = \
ProjectSettings.globalize_path(_get_wrapper_script_cache_path())
var startup_command : Array = [
"xterm", "-e",
python_exe_path,
wrapper_script_path,
"--script", python_script_path,
"--port", open_port,
"--parent_pid", OS.get_process_id()]
print("startup command: ", startup_command)
_external_process_pid = OS.create_process(
startup_command[0], startup_command.slice(1), true)
print("external process: ", _external_process_pid)
func stop_process():
if _external_process_pid != -1:
OS.kill(_external_process_pid)
_external_process_pid = -1
# Clean up server and communication sockets.
if _server_packet_socket:
_server_packet_socket.stop()
_server_packet_socket = null
if communication_packet_socket:
communication_packet_socket.stop()
communication_packet_socket = null
func call_rpc_async(method : String, args : Variant, callback = null) -> int: func call_rpc_async(method : String, args : Variant, callback = null) -> int:
assert((args is Dictionary) or (args is Array)) assert((args is Dictionary) or (args is Array))
@ -165,11 +198,11 @@ func call_rpc_sync(method : String, args : Variant):
func poll(): func poll():
# Hand-off between listening socket and actual communications socket. # Hand-off between listening socket and actual communications socket.
if server_packet_socket: if _server_packet_socket:
communication_packet_socket = server_packet_socket.get_next_server_connection() communication_packet_socket = _server_packet_socket.get_next_server_connection()
if communication_packet_socket: if communication_packet_socket:
server_packet_socket.stop() _server_packet_socket.stop()
server_packet_socket = null _server_packet_socket = null
if communication_packet_socket: if communication_packet_socket:

View File

@ -96,6 +96,8 @@ func _pad_to_512(x : int) -> int:
#endregion #endregion
#region Public API
func close() -> Error: func close() -> Error:
_internal_file_list = [] _internal_file_list = []
_reader.close() _reader.close()
@ -324,3 +326,5 @@ func unpack_file(dest_path : String, filename : String, overwrite : bool = false
record.mode, record.mode,
ProjectSettings.globalize_path(full_dest_path) ]) ProjectSettings.globalize_path(full_dest_path) ])
assert(err != -1) assert(err != -1)
#endregion

View File

@ -0,0 +1,4 @@
TODO
- How to use
- How to package Python stuff

View File

@ -0,0 +1,21 @@
The big ones:
- Handle bundling of the actual Python modules we want to use.
- First-time setup of requirements (pip, etc).
x Remove dependency on psutil.
- Clean up removal of psutil.
- Remove xterm dependency, or make it like a debug-only thing.
- Test on WINE/Windows.
- Documentation.
- how to use .kiri_export_python
- Un-thread the GDScript side of PacketSocket.
- Fix whatever this is:
SCRIPT ERROR: Assertion failed.
at: KiriPacketSocket._notification (res://addons/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd:70)
WARNING: A Thread object is being destroyed without its completion having been realized.
Please call wait_to_finish() on it to ensure correct cleanup.
at: ~Thread (core/os/thread.cpp:102)
- remove KiriPythonRPCWrapper_start.py
- remove test_rpc.py

View File

@ -22,7 +22,7 @@ window/vsync/vsync_mode=0
[editor_plugins] [editor_plugins]
enabled=PackedStringArray("res://addons/kiripythonrpcwrapper/plugin.cfg") enabled=PackedStringArray("res://addons/KiriPythonRPCWrapper/plugin.cfg")
[rendering] [rendering]