More stuff.

KiriJSONRPC.gd

@@ -1,3 +1,6 @@
+# FIXME: Remove this. I think we can nuke this entire class because we don't
+# need it anymore.
+
 # KiriJSONRPC
 #
 # This just wraps JSONRPC and adds a little more sanity-checking, like

TestPythonInExport.gd

@@ -1,17 +1,13 @@
 extends Node
 
 func _ready():
-	var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
 
-	bw.unpack_python()
+	var pw : KiriPythonWrapperInstance = KiriPythonWrapperInstance.new(
+		"/storage/git2/GodotJSONRPCTest/addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/test_module/__init__.py")
 
-	var out = []
+	pw.setup_python()
-	var ret = OS.execute(
+	pw.start_process()
-		bw.get_runtime_python_executable_system_path(),
-		["--version"], out, true)
 
-	print("Ret: ", ret)
+	var ret = pw.call_rpc_sync("func_to_call", ["test string whatever blah"])
-	print("Out: ", out)
+	print(ret)
-	
-	pass
 

addons/KiriPythonRPCWrapper/KiriPythonBuildExportPlugin.gd

@@ -1,13 +1,20 @@
+# Python build export plugin
+#
+# This just makes sure that the specific Python build for whatever platform we
+# need gets bundled into the build for that platform, so that it can be unpacked
+# and used later by KiriPythonBuildWrangler.
+
 @tool
 extends EditorExportPlugin
 class_name KiriPythonBuildExportPlugin
 
+func _get_name() -> String:
+	return "KiriPythonBuildExportPlugin"
+
 func _export_begin(
 	features : PackedStringArray, is_debug : bool,
 	path : String, flags : int):
 
-	print("features: ", features)
-
 	var build_wrangler : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
 
 	var platform_list = []
@@ -26,5 +33,26 @@ func _export_begin(
 		for arch in arch_list:
 			var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch)
 			var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export)
-			print("Adding file: ", archive_to_export, " ", len(file_contents))
 			add_file(archive_to_export, file_contents, false)
+
+	# Make sure all the RPC wrapper scripts make it in.
+	var script_path : String = get_script().resource_path
+	var script_dir : String = script_path.get_base_dir()
+
+	# Actually add all the files.
+	var extra_python_files = build_wrangler.get_extra_scripts_list()
+	for extra_python_file : String in extra_python_files:
+		var file_bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_python_file)
+		add_file(extra_python_file, file_bytes, false)
+
+	# Add the list of Python files as its own file so we know what to extract so
+	# it's visible to Python.
+	var python_wrapper_manifest_str : String = JSON.stringify(extra_python_files, "    ")
+	var python_wrapper_manifest_bytes : PackedByteArray = \
+		python_wrapper_manifest_str.to_utf8_buffer()
+	var python_wrapper_manifset_path = script_dir.path_join(
+		"KiriPythonWrapperPythonFiles.json")
+	add_file(python_wrapper_manifset_path, python_wrapper_manifest_bytes, false)
+
+
+

addons/KiriPythonRPCWrapper/KiriPythonBuildWrangler.gd

@@ -1,3 +1,7 @@
+# Python build wrangler
+#
+# This handles extracting and juggling standalone Python builds per-platform.
+
 extends RefCounted
 class_name KiriPythonBuildWrangler
 
@@ -217,7 +221,7 @@ func unpack_python(overwrite : bool = false):
 
 	# Open archive.
 	var python_archive_path : String = _detect_archive_for_runtime()
-	var reader : TARReader = TARReader.new()
+	var reader : KiriTARReader = KiriTARReader.new()
 	var err : Error = reader.open(python_archive_path)
 	assert(err == OK)
 
@@ -230,4 +234,66 @@ func unpack_python(overwrite : bool = false):
 
 # TODO: Clear cache function. Uninstall Python, etc.
 
+func get_extra_scripts_list() -> Array:
+
+	var script_path : String = get_script().resource_path
+	var script_dir : String = script_path.get_base_dir()
+	var python_wrapper_manifset_path = script_dir.path_join(
+		"KiriPythonWrapperPythonFiles.json")
+	
+	# If this is running an actual build, we'll just return the manifest here.
+	if FileAccess.file_exists(python_wrapper_manifset_path):
+		return load(python_wrapper_manifset_path).data
+
+	# If it's not running an actual build, we need to scan for extra Python
+	# files.
+	
+	# First pass: Find all the .kiri_export_python markers in the entire project
+	# tree.
+	var extra_python_files : Array = []
+	var scan_dir_list = ["res://"]
+	var verified_script_bundles = []
+	while len(scan_dir_list):
+		var current_dir : String = scan_dir_list.pop_front()
+		var da : DirAccess = DirAccess.open(current_dir)
+
+		if da.file_exists(".kiri_export_python"):
+			verified_script_bundles.append(current_dir)
+		else:
+
+			# Add all directories to the scan list.
+			da.include_navigational = false
+			var dir_list = da.get_directories()
+			for dir in dir_list:
+				if dir == "__pycache__":
+					continue
+				scan_dir_list.append(current_dir.path_join(dir))
+
+	# Second pass: Add everything under a directory containing a
+	# .kiri_export_python marker.
+	scan_dir_list = verified_script_bundles
+	while len(scan_dir_list):
+		var current_dir : String = scan_dir_list.pop_front()
+		var da : DirAccess = DirAccess.open(current_dir)
+
+		# Add all directories to the scan list.
+		da.include_navigational = false
+		var dir_list = da.get_directories()
+		for dir in dir_list:
+			if dir == "__pycache__":
+				continue
+			scan_dir_list.append(current_dir.path_join(dir))
+	
+		# Add all Python files.
+		var file_list = da.get_files()
+		for file in file_list:
+			var full_file = current_dir.path_join(file)
+			extra_python_files.append(full_file)
+
+	## FIXME: Remove this.
+	#for f in extra_python_files:
+		#print("Extra file: ", f)
+
+	return extra_python_files
+
 #endregion

addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd

@@ -1,3 +1,10 @@
+# KiriPacketSocket
+#
+# GDScript version of the KiriPacketSocket Python module. Basically just copied
+# the code over and reformatted it. Error handling and some other behaviors are
+# different due to differences in how Python and GDScript handle exceptions and
+# others.
+
 extends RefCounted
 class_name KiriPacketSocket
 

addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/__init__.py

@@ -4,42 +4,47 @@ import importlib.util
 import sys
 import argparse
 import time
-import psutil
+# import psutil
 import json
 
 import KiriPacketSocket
 
-# Parse arguments
+# This whole thing being in a try/except is just so we can catch
-arg_parser = argparse.ArgumentParser(
+# errors and see them before the terminal window closes.
+# try:
+if True:
+
+    # Parse arguments
+    arg_parser = argparse.ArgumentParser(
         prog="KiriPythonRPCWrapper",
         description="Wrapper for Python modules to RPCs from Godot.",
         epilog="")
 
-arg_parser.add_argument("--script", type=str, required=True)
+    arg_parser.add_argument("--script", type=str, required=True)
-arg_parser.add_argument("--port", type=int, required=True)
+    arg_parser.add_argument("--port", type=int, required=True)
-arg_parser.add_argument("--parent_pid", type=int, required=True)
+    arg_parser.add_argument("--parent_pid", type=int, required=True)
 
-args = arg_parser.parse_args()
+    args = arg_parser.parse_args()
 
 
-# module_path = "../KiriPacketSocket/__init__.py"
+    # module_path = "../KiriPacketSocket/__init__.py"
-# module_name = "KiriPacketSocket"
+    # module_name = "KiriPacketSocket"
 
-module_path = args.script
+    module_path = args.script
-module_name = ""
+    module_name = ""
 
-# Attempt to load the module.
+    # Attempt to load the module.
-module_spec = importlib.util.spec_from_file_location(
+    module_spec = importlib.util.spec_from_file_location(
         module_name, module_path)
-module = importlib.util.module_from_spec(module_spec)
+    module = importlib.util.module_from_spec(module_spec)
-module_spec.loader.exec_module(module)
+    module_spec.loader.exec_module(module)
 
-# This will be all the functions we find in the module that don't
+    # This will be all the functions we find in the module that don't
-# start with "_".
+    # start with "_".
-known_entrypoints = {}
+    known_entrypoints = {}
 
-# Scan the module for "public" functions.
+    # Scan the module for "public" functions.
-for entrypoint in dir(module):
+    for entrypoint in dir(module):
 
         # Skip anything starting with "_". Probably not meant to be
         # exposed.
@@ -52,21 +57,21 @@ for entrypoint in dir(module):
         if callable(attr):
             known_entrypoints[entrypoint] = attr
 
-# Connect to server.
+    # Connect to server.
-packet_socket =  KiriPacketSocket.PacketSocket()
+    packet_socket =  KiriPacketSocket.PacketSocket()
-packet_socket.start_client(("127.0.0.1", args.port))
+    packet_socket.start_client(("127.0.0.1", args.port))
-while packet_socket.get_state() == packet_socket.SocketState.CONNECTING:
+    while packet_socket.get_state() == packet_socket.SocketState.CONNECTING:
         time.sleep(0.001)
 
-if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
+    if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
         packet_socket.stop()
         raise Exception("Failed to connect to RPC host.")
 
-print("Starting packet processing.")
+    print("Starting packet processing.")
 
 
 
-def send_error_response(code, message, request_id):
+    def send_error_response(code, message, request_id):
         ret_dict = {
             "jsonrpc" : "2.0",
             "error" : {
@@ -78,7 +83,7 @@ def send_error_response(code, message, request_id):
         ret_dict_json = json.dumps(ret_dict)
         packet_socket.send_packet(ret_dict_json.encode("utf-8"))
 
-def send_response(result, request_id):
+    def send_response(result, request_id):
         try:
             ret_dict = {
                 "jsonrpc" : "2.0",
@@ -90,18 +95,18 @@ def send_response(result, request_id):
         except Exception as e:
             send_error_response(-32603, "Error sending result: " + str(e), request_id)
 
-# Start processing packets.
+    # Start processing packets.
-while True:
+    while True:
 
         # Shutdown when we lose connection to host.
         if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
             packet_socket.stop()
             raise Exception("Disconnected from RPC host.")
 
-    # Watch parent PID so we can clean up when needed.
+        # # Watch parent PID so we can clean up when needed.
-    if not psutil.pid_exists(args.parent_pid):
+        # if not psutil.pid_exists(args.parent_pid):
-        packet_socket.stop()
+        #     packet_socket.stop()
-        raise Exception("RPC host process died")
+        #     raise Exception("RPC host process died")
 
         next_packet = packet_socket.get_next_packet()
         while next_packet:
@@ -147,6 +152,10 @@ while True:
 
         time.sleep(0.0001)
 
+# except Exception as e:
+#     sys.stderr.write(e)
+#     time.sleep(5)
+#     raise e
 
 
 

addons/KiriPythonRPCWrapper/KiriPythonWrapperInstance.gd

@@ -1,103 +1,11 @@
 extends RefCounted
 class_name KiriPythonWrapperInstance
 
-var external_process_pid = -1
-
-var server_packet_socket : KiriPacketSocket = null
-var communication_packet_socket : KiriPacketSocket = null
-
-var python_script_path : String = ""
-
 enum KiriPythonWrapperStatus {
 	STATUS_RUNNING,
 	STATUS_STOPPED
 }
 
-func _init(python_file_path : String):
-	python_script_path = python_file_path
-
-func _get_python_executable():
-	# FIXME: Adjust per-OS. Maybe test a few locations.
-	return "/usr/bin/python3"
-
-func _get_wrapper_script():
-	# FIXME: Paths will be different for builds.
-	var script_path = self.get_script().get_path()
-	var script_dirname = script_path.get_base_dir()
-	return ProjectSettings.globalize_path( \
-		script_dirname + "/KiriPythonRPCWrapper_start.py")
-
-func get_status():
-
-	if external_process_pid == -1:
-		return KiriPythonWrapperStatus.STATUS_STOPPED
-
-	if not OS.is_process_running(external_process_pid):
-		return KiriPythonWrapperStatus.STATUS_STOPPED
-
-	return KiriPythonWrapperStatus.STATUS_RUNNING
-
-func start_process():
-
-	# FIXME: Make sure we don't have one running.
-
-	var open_port = 9500
-	
-	assert(not server_packet_socket)
-	server_packet_socket = KiriPacketSocket.new()
-	while true:
-		server_packet_socket.start_server(["127.0.0.1", open_port])
-		
-		# Wait for the server to start.
-		while server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
-			OS.delay_usec(1)
-		
-		# If we're successfully listening, then we found a port to use and we
-		# don't need to loop anymore.
-		if server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
-			break
-		
-		# This port is busy. Try the next one.
-		server_packet_socket.stop()
-		open_port += 1
-
-	print("Port: ", open_port)
-	
-	var python_exe_path : String = _get_python_executable()
-	var wrapper_script_path : String = _get_wrapper_script()
-
-	var startup_command : Array = [
-		"xterm", "-e",
-		python_exe_path,
-		wrapper_script_path,
-		"--script", python_script_path,
-		"--port", open_port,
-		"--parent_pid", OS.get_process_id()]
-
-	print("startup command: ", startup_command)
-
-	external_process_pid = OS.create_process(
-		startup_command[0], startup_command.slice(1), true)
-
-	print("external process: ", external_process_pid)
-
-func stop_process():
-
-	if external_process_pid != -1:
-		OS.kill(external_process_pid)
-		external_process_pid = -1
-		
-
-	# Clean up server and communication sockets.
-	if server_packet_socket:
-		server_packet_socket.stop()
-		server_packet_socket = null
-
-	if communication_packet_socket:
-		communication_packet_socket.stop()
-		communication_packet_socket = null
-
-
 class KiriPythonWrapperActiveRequest:
 	
 	enum KiriPythonWrapperActiveRequestState {
@@ -117,6 +25,131 @@ class KiriPythonWrapperActiveRequest:
 var _active_request_queue = {}
 var _request_counter = 0
 
+var _server_packet_socket : KiriPacketSocket = null
+var communication_packet_socket : KiriPacketSocket = null
+
+var python_script_path : String = ""
+
+var _build_wrangler : KiriPythonBuildWrangler = null
+
+var _external_process_pid = -1
+
+
+func _init(python_file_path : String):
+	_build_wrangler = KiriPythonBuildWrangler.new()
+	python_script_path = python_file_path
+
+func _get_python_executable():
+	return _build_wrangler.get_runtime_python_executable_system_path()
+
+func _get_wrapper_script():
+	# FIXME: Paths will be different for builds.
+	var script_path = self.get_script().get_path()
+	var script_dirname = script_path.get_base_dir()
+	return ProjectSettings.globalize_path( \
+		script_dirname + "/KiriPythonRPCWrapper_start.py")
+
+func _get_wrapper_cache_path() -> String:
+	return _build_wrangler._get_cache_path_godot().path_join("KiriPythonRPCWrapper")
+
+func _get_wrapper_script_cache_path() -> String:
+	return _get_wrapper_cache_path().path_join("addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/__init__.py")
+
+func setup_python():
+
+	# Unpack base Python build.
+	_build_wrangler.unpack_python(false)
+
+	# Unpack Python wrapper.
+	var extra_scripts = _build_wrangler.get_extra_scripts_list()
+	print(extra_scripts)
+	for extra_script : String in extra_scripts:
+		
+		# Chop off the "res://".
+		var extra_script_relative : String = extra_script.substr(len("res://"))
+
+		# Some other path wrangling.
+		var extraction_path : String = _get_wrapper_cache_path().path_join(extra_script_relative)
+		var extraction_path_dir : String = extraction_path.get_base_dir()
+		
+		# Make the dir.
+		DirAccess.make_dir_recursive_absolute(extraction_path_dir)
+		
+		# Extract the file.
+		var bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_script)
+		FileAccess.open(extraction_path, FileAccess.WRITE).store_buffer(bytes)
+
+
+func get_status():
+
+	if _external_process_pid == -1:
+		return KiriPythonWrapperStatus.STATUS_STOPPED
+
+	if not OS.is_process_running(_external_process_pid):
+		return KiriPythonWrapperStatus.STATUS_STOPPED
+
+	return KiriPythonWrapperStatus.STATUS_RUNNING
+
+func start_process():
+
+	# FIXME: Make sure we don't have one running.
+
+	var open_port = 9500
+	
+	assert(not _server_packet_socket)
+	_server_packet_socket = KiriPacketSocket.new()
+	while true:
+		_server_packet_socket.start_server(["127.0.0.1", open_port])
+		
+		# Wait for the server to start.
+		while _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
+			OS.delay_usec(1)
+		
+		# If we're successfully listening, then we found a port to use and we
+		# don't need to loop anymore.
+		if _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
+			break
+		
+		# This port is busy. Try the next one.
+		_server_packet_socket.stop()
+		open_port += 1
+
+	print("Port: ", open_port)
+	
+	var python_exe_path : String = _get_python_executable()
+	var wrapper_script_path : String = \
+		ProjectSettings.globalize_path(_get_wrapper_script_cache_path())
+
+	var startup_command : Array = [
+		"xterm", "-e",
+		python_exe_path,
+		wrapper_script_path,
+		"--script", python_script_path,
+		"--port", open_port,
+		"--parent_pid", OS.get_process_id()]
+
+	print("startup command: ", startup_command)
+
+	_external_process_pid = OS.create_process(
+		startup_command[0], startup_command.slice(1), true)
+
+	print("external process: ", _external_process_pid)
+
+func stop_process():
+
+	if _external_process_pid != -1:
+		OS.kill(_external_process_pid)
+		_external_process_pid = -1
+
+	# Clean up server and communication sockets.
+	if _server_packet_socket:
+		_server_packet_socket.stop()
+		_server_packet_socket = null
+
+	if communication_packet_socket:
+		communication_packet_socket.stop()
+		communication_packet_socket = null
+
 func call_rpc_async(method : String, args : Variant, callback = null) -> int:
 	
 	assert((args is Dictionary) or (args is Array))
@@ -165,11 +198,11 @@ func call_rpc_sync(method : String, args : Variant):
 func poll():
 
 	# Hand-off between listening socket and actual communications socket.
-	if server_packet_socket:
+	if _server_packet_socket:
-		communication_packet_socket = server_packet_socket.get_next_server_connection()
+		communication_packet_socket = _server_packet_socket.get_next_server_connection()
 		if communication_packet_socket:
-			server_packet_socket.stop()
+			_server_packet_socket.stop()
-			server_packet_socket = null
+			_server_packet_socket = null
 	
 	if communication_packet_socket:
 		

addons/KiriPythonRPCWrapper/KiriTARReader.gd

@@ -96,6 +96,8 @@ func _pad_to_512(x : int) -> int:
 
 #endregion
 
+#region Public API
+
 func close() -> Error:
 	_internal_file_list = []
 	_reader.close()
@@ -324,3 +326,5 @@ func unpack_file(dest_path : String, filename : String, overwrite : bool = false
 						record.mode,
 						ProjectSettings.globalize_path(full_dest_path) ])
 					assert(err != -1)
+
+#endregion

addons/KiriPythonRPCWrapper/README.md

@@ -0,0 +1,4 @@
+TODO
+	- How to use
+	- How to package Python stuff
+

addons/KiriPythonRPCWrapper/TODO.md

@@ -0,0 +1,21 @@
+The big ones:
+	- Handle bundling of the actual Python modules we want to use.
+	- First-time setup of requirements (pip, etc).
+	x Remove dependency on psutil.
+	- Clean up removal of psutil.
+	- Remove xterm dependency, or make it like a debug-only thing.
+	- Test on WINE/Windows.
+	- Documentation.
+		- how to use .kiri_export_python
+
+	- Un-thread the GDScript side of PacketSocket.
+
+	- Fix whatever this is:
+		SCRIPT ERROR: Assertion failed.
+		  at: KiriPacketSocket._notification (res://addons/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd:70)
+WARNING: A Thread object is being destroyed without its completion having been realized.
+Please call wait_to_finish() on it to ensure correct cleanup.
+	 at: ~Thread (core/os/thread.cpp:102)
+
+	- remove KiriPythonRPCWrapper_start.py
+	- remove test_rpc.py

project.godot

@@ -22,7 +22,7 @@ window/vsync/vsync_mode=0
 
 [editor_plugins]
 
-enabled=PackedStringArray("res://addons/kiripythonrpcwrapper/plugin.cfg")
+enabled=PackedStringArray("res://addons/KiriPythonRPCWrapper/plugin.cfg")
 
 [rendering]