ExpiredPopsicle/GodotPythonJSONRPC
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More stuff.

Browse Source
This commit is contained in:
Kiri 2024-07-14 12:58:20 -07:00
parent bdd56c30d1
commit 4cad6a574f
16 changed files with 388 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
KiriJSONRPC.gd
View File

@ -1,3 +1,6 @@
# FIXME: Remove this. I think we can nuke this entire class because we don't
# need it anymore.
# KiriJSONRPC
#
# This just wraps JSONRPC and adds a little more sanity-checking, like

16
TestPythonInExport.gd
View File

@ -1,17 +1,13 @@
extends Node
func _ready():
var bw : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
bw.unpack_python()
var pw : KiriPythonWrapperInstance = KiriPythonWrapperInstance.new(
"/storage/git2/GodotJSONRPCTest/addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/test_module/__init__.py")
var out = []
var ret = OS.execute(
bw.get_runtime_python_executable_system_path(),
["--version"], out, true)
pw.setup_python()
pw.start_process()
print("Ret: ", ret)
print("Out: ", out)
pass
var ret = pw.call_rpc_sync("func_to_call", ["test string whatever blah"])
print(ret)

34
addons/KiriPythonRPCWrapper/KiriPythonBuildExportPlugin.gd
View File

@ -1,13 +1,20 @@
# Python build export plugin
#
# This just makes sure that the specific Python build for whatever platform we
# need gets bundled into the build for that platform, so that it can be unpacked
# and used later by KiriPythonBuildWrangler.
@tool
extends EditorExportPlugin
class_name KiriPythonBuildExportPlugin
func _get_name() -> String:
return "KiriPythonBuildExportPlugin"
func _export_begin(
features : PackedStringArray, is_debug : bool,
path : String, flags : int):
print("features: ", features)
var build_wrangler : KiriPythonBuildWrangler = KiriPythonBuildWrangler.new()
var platform_list = []
@ -26,5 +33,26 @@ func _export_begin(
for arch in arch_list:
var archive_to_export = build_wrangler._detect_archive_for_build(platform, arch)
var file_contents : PackedByteArray = FileAccess.get_file_as_bytes(archive_to_export)
print("Adding file: ", archive_to_export, " ", len(file_contents))
add_file(archive_to_export, file_contents, false)
# Make sure all the RPC wrapper scripts make it in.
var script_path : String = get_script().resource_path
var script_dir : String = script_path.get_base_dir()
# Actually add all the files.
var extra_python_files = build_wrangler.get_extra_scripts_list()
for extra_python_file : String in extra_python_files:
var file_bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_python_file)
add_file(extra_python_file, file_bytes, false)
# Add the list of Python files as its own file so we know what to extract so
# it's visible to Python.
var python_wrapper_manifest_str : String = JSON.stringify(extra_python_files, " ")
var python_wrapper_manifest_bytes : PackedByteArray = \
python_wrapper_manifest_str.to_utf8_buffer()
var python_wrapper_manifset_path = script_dir.path_join(
"KiriPythonWrapperPythonFiles.json")
add_file(python_wrapper_manifset_path, python_wrapper_manifest_bytes, false)

68
addons/KiriPythonRPCWrapper/KiriPythonBuildWrangler.gd
View File

@ -1,3 +1,7 @@
# Python build wrangler
#
# This handles extracting and juggling standalone Python builds per-platform.
extends RefCounted
class_name KiriPythonBuildWrangler
@ -217,7 +221,7 @@ func unpack_python(overwrite : bool = false):
# Open archive.
var python_archive_path : String = _detect_archive_for_runtime()
var reader : TARReader = TARReader.new()
var reader : KiriTARReader = KiriTARReader.new()
var err : Error = reader.open(python_archive_path)
assert(err == OK)
@ -230,4 +234,66 @@ func unpack_python(overwrite : bool = false):
# TODO: Clear cache function. Uninstall Python, etc.
func get_extra_scripts_list() -> Array:
var script_path : String = get_script().resource_path
var script_dir : String = script_path.get_base_dir()
var python_wrapper_manifset_path = script_dir.path_join(
"KiriPythonWrapperPythonFiles.json")
# If this is running an actual build, we'll just return the manifest here.
if FileAccess.file_exists(python_wrapper_manifset_path):
return load(python_wrapper_manifset_path).data
# If it's not running an actual build, we need to scan for extra Python
# files.
# First pass: Find all the .kiri_export_python markers in the entire project
# tree.
var extra_python_files : Array = []
var scan_dir_list = ["res://"]
var verified_script_bundles = []
while len(scan_dir_list):
var current_dir : String = scan_dir_list.pop_front()
var da : DirAccess = DirAccess.open(current_dir)
if da.file_exists(".kiri_export_python"):
verified_script_bundles.append(current_dir)
else:
# Add all directories to the scan list.
da.include_navigational = false
var dir_list = da.get_directories()
for dir in dir_list:
if dir == "__pycache__":
continue
scan_dir_list.append(current_dir.path_join(dir))
# Second pass: Add everything under a directory containing a
# .kiri_export_python marker.
scan_dir_list = verified_script_bundles
while len(scan_dir_list):
var current_dir : String = scan_dir_list.pop_front()
var da : DirAccess = DirAccess.open(current_dir)
# Add all directories to the scan list.
da.include_navigational = false
var dir_list = da.get_directories()
for dir in dir_list:
if dir == "__pycache__":
continue
scan_dir_list.append(current_dir.path_join(dir))
# Add all Python files.
var file_list = da.get_files()
for file in file_list:
var full_file = current_dir.path_join(file)
extra_python_files.append(full_file)
## FIXME: Remove this.
#for f in extra_python_files:
#print("Extra file: ", f)
return extra_python_files
#endregion

0
addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/.kiri_export_python Normal file
View File

0
addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/.kiri_export_python Normal file
View File

7
addons/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd → addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd
View File

@ -1,3 +1,10 @@
# KiriPacketSocket
#
# GDScript version of the KiriPacketSocket Python module. Basically just copied
# the code over and reformatted it. Error handling and some other behaviors are
# different due to differences in how Python and GDScript handle exceptions and
# others.
extends RefCounted
class_name KiriPacketSocket

0
addons/KiriPythonRPCWrapper/KiriPacketSocket/LICENSE.md → addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/LICENSE.md
View File

0
addons/KiriPythonRPCWrapper/KiriPacketSocket/README.md → addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/README.md
View File

0
addons/KiriPythonRPCWrapper/KiriPacketSocket/__init__.py → addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/KiriPacketSocket/__init__.py
View File

77
addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/__init__.py
View File

@ -4,42 +4,47 @@ import importlib.util
import sys
import argparse
import time
import psutil
# import psutil
import json
import KiriPacketSocket
# Parse arguments
arg_parser = argparse.ArgumentParser(
# This whole thing being in a try/except is just so we can catch
# errors and see them before the terminal window closes.
# try:
if True:
# Parse arguments
arg_parser = argparse.ArgumentParser(
prog="KiriPythonRPCWrapper",
description="Wrapper for Python modules to RPCs from Godot.",
epilog="")
arg_parser.add_argument("--script", type=str, required=True)
arg_parser.add_argument("--port", type=int, required=True)
arg_parser.add_argument("--parent_pid", type=int, required=True)
arg_parser.add_argument("--script", type=str, required=True)
arg_parser.add_argument("--port", type=int, required=True)
arg_parser.add_argument("--parent_pid", type=int, required=True)
args = arg_parser.parse_args()
args = arg_parser.parse_args()
# module_path = "../KiriPacketSocket/__init__.py"
# module_name = "KiriPacketSocket"
# module_path = "../KiriPacketSocket/__init__.py"
# module_name = "KiriPacketSocket"
module_path = args.script
module_name = ""
module_path = args.script
module_name = ""
# Attempt to load the module.
module_spec = importlib.util.spec_from_file_location(
# Attempt to load the module.
module_spec = importlib.util.spec_from_file_location(
module_name, module_path)
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
module = importlib.util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
# This will be all the functions we find in the module that don't
# start with "_".
known_entrypoints = {}
# This will be all the functions we find in the module that don't
# start with "_".
known_entrypoints = {}
# Scan the module for "public" functions.
for entrypoint in dir(module):
# Scan the module for "public" functions.
for entrypoint in dir(module):
# Skip anything starting with "_". Probably not meant to be
# exposed.
@ -52,21 +57,21 @@ for entrypoint in dir(module):
if callable(attr):
known_entrypoints[entrypoint] = attr
# Connect to server.
packet_socket = KiriPacketSocket.PacketSocket()
packet_socket.start_client(("127.0.0.1", args.port))
while packet_socket.get_state() == packet_socket.SocketState.CONNECTING:
# Connect to server.
packet_socket = KiriPacketSocket.PacketSocket()
packet_socket.start_client(("127.0.0.1", args.port))
while packet_socket.get_state() == packet_socket.SocketState.CONNECTING:
time.sleep(0.001)
if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
packet_socket.stop()
raise Exception("Failed to connect to RPC host.")
print("Starting packet processing.")
print("Starting packet processing.")
def send_error_response(code, message, request_id):
def send_error_response(code, message, request_id):
ret_dict = {
"jsonrpc" : "2.0",
"error" : {
@ -78,7 +83,7 @@ def send_error_response(code, message, request_id):
ret_dict_json = json.dumps(ret_dict)
packet_socket.send_packet(ret_dict_json.encode("utf-8"))
def send_response(result, request_id):
def send_response(result, request_id):
try:
ret_dict = {
"jsonrpc" : "2.0",
@ -90,18 +95,18 @@ def send_response(result, request_id):
except Exception as e:
send_error_response(-32603, "Error sending result: " + str(e), request_id)
# Start processing packets.
while True:
# Start processing packets.
while True:
# Shutdown when we lose connection to host.
if packet_socket.get_state() != packet_socket.SocketState.CONNECTED:
packet_socket.stop()
raise Exception("Disconnected from RPC host.")
# Watch parent PID so we can clean up when needed.
if not psutil.pid_exists(args.parent_pid):
packet_socket.stop()
raise Exception("RPC host process died")
# # Watch parent PID so we can clean up when needed.
# if not psutil.pid_exists(args.parent_pid):
# packet_socket.stop()
# raise Exception("RPC host process died")
next_packet = packet_socket.get_next_packet()
while next_packet:
@ -147,6 +152,10 @@ while True:
time.sleep(0.0001)
# except Exception as e:
# sys.stderr.write(e)
# time.sleep(5)
# raise e

225
addons/KiriPythonRPCWrapper/KiriPythonWrapperInstance.gd
View File

@ -1,103 +1,11 @@
extends RefCounted
class_name KiriPythonWrapperInstance
var external_process_pid = -1
var server_packet_socket : KiriPacketSocket = null
var communication_packet_socket : KiriPacketSocket = null
var python_script_path : String = ""
enum KiriPythonWrapperStatus {
STATUS_RUNNING,
STATUS_STOPPED
}
func _init(python_file_path : String):
python_script_path = python_file_path
func _get_python_executable():
# FIXME: Adjust per-OS. Maybe test a few locations.
return "/usr/bin/python3"
func _get_wrapper_script():
# FIXME: Paths will be different for builds.
var script_path = self.get_script().get_path()
var script_dirname = script_path.get_base_dir()
return ProjectSettings.globalize_path( \
script_dirname + "/KiriPythonRPCWrapper_start.py")
func get_status():
if external_process_pid == -1:
return KiriPythonWrapperStatus.STATUS_STOPPED
if not OS.is_process_running(external_process_pid):
return KiriPythonWrapperStatus.STATUS_STOPPED
return KiriPythonWrapperStatus.STATUS_RUNNING
func start_process():
# FIXME: Make sure we don't have one running.
var open_port = 9500
assert(not server_packet_socket)
server_packet_socket = KiriPacketSocket.new()
while true:
server_packet_socket.start_server(["127.0.0.1", open_port])
# Wait for the server to start.
while server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
OS.delay_usec(1)
# If we're successfully listening, then we found a port to use and we
# don't need to loop anymore.
if server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
break
# This port is busy. Try the next one.
server_packet_socket.stop()
open_port += 1
print("Port: ", open_port)
var python_exe_path : String = _get_python_executable()
var wrapper_script_path : String = _get_wrapper_script()
var startup_command : Array = [
"xterm", "-e",
python_exe_path,
wrapper_script_path,
"--script", python_script_path,
"--port", open_port,
"--parent_pid", OS.get_process_id()]
print("startup command: ", startup_command)
external_process_pid = OS.create_process(
startup_command[0], startup_command.slice(1), true)
print("external process: ", external_process_pid)
func stop_process():
if external_process_pid != -1:
OS.kill(external_process_pid)
external_process_pid = -1
# Clean up server and communication sockets.
if server_packet_socket:
server_packet_socket.stop()
server_packet_socket = null
if communication_packet_socket:
communication_packet_socket.stop()
communication_packet_socket = null
class KiriPythonWrapperActiveRequest:
enum KiriPythonWrapperActiveRequestState {
@ -117,6 +25,131 @@ class KiriPythonWrapperActiveRequest:
var _active_request_queue = {}
var _request_counter = 0
var _server_packet_socket : KiriPacketSocket = null
var communication_packet_socket : KiriPacketSocket = null
var python_script_path : String = ""
var _build_wrangler : KiriPythonBuildWrangler = null
var _external_process_pid = -1
func _init(python_file_path : String):
_build_wrangler = KiriPythonBuildWrangler.new()
python_script_path = python_file_path
func _get_python_executable():
return _build_wrangler.get_runtime_python_executable_system_path()
func _get_wrapper_script():
# FIXME: Paths will be different for builds.
var script_path = self.get_script().get_path()
var script_dirname = script_path.get_base_dir()
return ProjectSettings.globalize_path( \
script_dirname + "/KiriPythonRPCWrapper_start.py")
func _get_wrapper_cache_path() -> String:
return _build_wrangler._get_cache_path_godot().path_join("KiriPythonRPCWrapper")
func _get_wrapper_script_cache_path() -> String:
return _get_wrapper_cache_path().path_join("addons/KiriPythonRPCWrapper/KiriPythonRPCWrapper/__init__.py")
func setup_python():
# Unpack base Python build.
_build_wrangler.unpack_python(false)
# Unpack Python wrapper.
var extra_scripts = _build_wrangler.get_extra_scripts_list()
print(extra_scripts)
for extra_script : String in extra_scripts:
# Chop off the "res://".
var extra_script_relative : String = extra_script.substr(len("res://"))
# Some other path wrangling.
var extraction_path : String = _get_wrapper_cache_path().path_join(extra_script_relative)
var extraction_path_dir : String = extraction_path.get_base_dir()
# Make the dir.
DirAccess.make_dir_recursive_absolute(extraction_path_dir)
# Extract the file.
var bytes : PackedByteArray = FileAccess.get_file_as_bytes(extra_script)
FileAccess.open(extraction_path, FileAccess.WRITE).store_buffer(bytes)
func get_status():
if _external_process_pid == -1:
return KiriPythonWrapperStatus.STATUS_STOPPED
if not OS.is_process_running(_external_process_pid):
return KiriPythonWrapperStatus.STATUS_STOPPED
return KiriPythonWrapperStatus.STATUS_RUNNING
func start_process():
# FIXME: Make sure we don't have one running.
var open_port = 9500
assert(not _server_packet_socket)
_server_packet_socket = KiriPacketSocket.new()
while true:
_server_packet_socket.start_server(["127.0.0.1", open_port])
# Wait for the server to start.
while _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_STARTING:
OS.delay_usec(1)
# If we're successfully listening, then we found a port to use and we
# don't need to loop anymore.
if _server_packet_socket.get_state() == KiriPacketSocket.KiriSocketState.SERVER_LISTENING:
break
# This port is busy. Try the next one.
_server_packet_socket.stop()
open_port += 1
print("Port: ", open_port)
var python_exe_path : String = _get_python_executable()
var wrapper_script_path : String = \
ProjectSettings.globalize_path(_get_wrapper_script_cache_path())
var startup_command : Array = [
"xterm", "-e",
python_exe_path,
wrapper_script_path,
"--script", python_script_path,
"--port", open_port,
"--parent_pid", OS.get_process_id()]
print("startup command: ", startup_command)
_external_process_pid = OS.create_process(
startup_command[0], startup_command.slice(1), true)
print("external process: ", _external_process_pid)
func stop_process():
if _external_process_pid != -1:
OS.kill(_external_process_pid)
_external_process_pid = -1
# Clean up server and communication sockets.
if _server_packet_socket:
_server_packet_socket.stop()
_server_packet_socket = null
if communication_packet_socket:
communication_packet_socket.stop()
communication_packet_socket = null
func call_rpc_async(method : String, args : Variant, callback = null) -> int:
assert((args is Dictionary) or (args is Array))
@ -165,11 +198,11 @@ func call_rpc_sync(method : String, args : Variant):
func poll():
# Hand-off between listening socket and actual communications socket.
if server_packet_socket:
communication_packet_socket = server_packet_socket.get_next_server_connection()
if _server_packet_socket:
communication_packet_socket = _server_packet_socket.get_next_server_connection()
if communication_packet_socket:
server_packet_socket.stop()
server_packet_socket = null
_server_packet_socket.stop()
_server_packet_socket = null
if communication_packet_socket:

4
addons/KiriPythonRPCWrapper/KiriTARReader.gd
View File

@ -96,6 +96,8 @@ func _pad_to_512(x : int) -> int:
#endregion
#region Public API
func close() -> Error:
_internal_file_list = []
_reader.close()
@ -324,3 +326,5 @@ func unpack_file(dest_path : String, filename : String, overwrite : bool = false
record.mode,
ProjectSettings.globalize_path(full_dest_path) ])
assert(err != -1)
#endregion

4
addons/KiriPythonRPCWrapper/README.md Normal file
View File

@ -0,0 +1,4 @@
TODO
- How to use
- How to package Python stuff

21
addons/KiriPythonRPCWrapper/TODO.md Normal file
View File

@ -0,0 +1,21 @@
The big ones:
- Handle bundling of the actual Python modules we want to use.
- First-time setup of requirements (pip, etc).
x Remove dependency on psutil.
- Clean up removal of psutil.
- Remove xterm dependency, or make it like a debug-only thing.
- Test on WINE/Windows.
- Documentation.
- how to use .kiri_export_python
- Un-thread the GDScript side of PacketSocket.
- Fix whatever this is:
SCRIPT ERROR: Assertion failed.
at: KiriPacketSocket._notification (res://addons/KiriPythonRPCWrapper/KiriPacketSocket/KiriPacketSocket.gd:70)
WARNING: A Thread object is being destroyed without its completion having been realized.
Please call wait_to_finish() on it to ensure correct cleanup.
at: ~Thread (core/os/thread.cpp:102)
- remove KiriPythonRPCWrapper_start.py
- remove test_rpc.py

2
project.godot
View File

@ -22,7 +22,7 @@ window/vsync/vsync_mode=0
[editor_plugins]
enabled=PackedStringArray("res://addons/kiripythonrpcwrapper/plugin.cfg")
enabled=PackedStringArray("res://addons/KiriPythonRPCWrapper/plugin.cfg")
[rendering]