chore(sdk): add prototype python wrapper for nexus

core/lang/core/interface_pb.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+ "C"
+ "unsafe"
+ "google.golang.org/protobuf/proto"
+ "github.com/wandb/wandb/core/pkg/service"
+ "github.com/wandb/wandb/core/pkg/gowandb/opts/runopts"
+)
+
+//export pbSessionSetup
+func pbSessionSetup() {
+ wandbcoreSetup()
+}
+
+//export pbSessionTeardown
+func pbSessionTeardown() {
+ // prob dont want this, we could share nexus across "sessions"
+ wandbcoreTeardown()
+}
+
+//export pbRunStart
+func pbRunStart() int {
+ options := []runopts.RunOption{}
+ wandbcoreSetup()
+ run, err := wandbSession.NewRun(options...)
+ if err != nil {
+ panic(err)
+ }
+ num := wandbRuns.Add(run)
+ return num
+}
+
+//export pbRunLog
+func pbRunLog(num int, cBuffer *C.char, cLength C.int) {
+ data := C.GoBytes(unsafe.Pointer(cBuffer), cLength)
+ // Unmarshal protobuf
+ msg := &service.DataRecord{}
+ if err := proto.Unmarshal(data, msg); err != nil {
+ return
+ }
+ // Process data (here simply prepending a string)
+ // fmt.Printf("PROTO %+v\n", msg)
+}
+
+//export pbRunFinish
+func pbRunFinish(num int) {
+ run := wandbRuns.Get(num)
+ run.Finish()
+ wandbRuns.Remove(num)
+}

core/lang/py/examples/setup-nexus-py.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+rm -rf nexus-py-1
+virtualenv nexus-py-1
+source nexus-py-1/bin/activate
+pip install --upgrade pip
+cd ../lib
+./build_proto.sh
+./build_lib.sh
+pip install -e .
+echo "Run:"
+echo "source nexus-py-1/bin/activate"

core/lang/py/examples/simple-newapi.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+import wandb
+
+"""
+Top level:
+new_api() -> API (? or sdk or library or core..or not)
+default_sdk
+default_session
+default_run
+(promote mothods from default_sdk, default_session - and maybe default_run to top level namespace?)
+
+API:
+ new_session -> Session
+
+Session:
+ login()
+ configure_auth()
+ new_run()
+ get_run() # might have mutable and readonly versions of the run? readonly by default?
+ # alternate, prefix with object type? run_new, run_get... dont love
+ # ? are api runs just like runapi --> can we log to a run from the public api? why not?
+
+Run:
+ log()
+ history() -> how does this work for a run in progress
+
+
+"""
+
+api = wandb.new_api()
+session = api.new_session()
+run = session.get_run()
+for row in run.history():
+ pass

core/lang/py/examples/simple-newsdk.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+import wandb
+
+wb = wandb.new_session()
+run = wb.new_run()
+run.log({"a": 1, "b": 2})
+run.finish()

core/lang/py/examples/simple.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+import wandb
+
+run = wandb.init()
+run.log({"a": 1, "b": 2})
+run.finish()

core/lang/py/lib/MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include wandb *.py
+recursive-include wandb *.sh
+recursive-include wandb *.so
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

core/lang/py/lib/build_lib.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+BASE=../../
+DEST=py/lib/
+cd $BASE/
+./scripts/base-build.sh
+mkdir -p $DEST/wandb/lib
+cp export/lib/libwandb_core.so $DEST/wandb/lib

core/lang/py/lib/build_proto.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+BASE=../../../..
+DEST=core/lang/py/lib/
+# cp $BASE/wandb/proto/*.proto wandb/proto/
+cd $BASE/
+protoc -I=. --python_out=$DEST wandb/proto/wandb_base.proto
+protoc -I=. --python_out=$DEST wandb/proto/wandb_internal.proto
+protoc -I=. --python_out=$DEST wandb/proto/wandb_telemetry.proto
+protoc -I=. --python_out=$DEST wandb/proto/wandb_settings.proto
+protoc -I=. --python_out=$DEST wandb/proto/wandb_server.proto

core/lang/py/lib/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ['setuptools>61']
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wandb"
+dynamic = ["version"]
+description = "A CLI and library for interacting with the Weights & Biases API."
+authors = [{ name = "Weights & Biases", email = "[email protected]" }]
+# readme = "package_readme.md"
+# license = { file = "LICENSE" }
+requires-python = ">=3.7"
+dependencies = [
+ "protobuf>=3.12.0,!=4.21.0,<5; python_version < '3.9' and sys_platform == 'linux'",
+ "protobuf>=3.15.0,!=4.21.0,<5; python_version == '3.9' and sys_platform == 'linux'",
+ "protobuf>=3.19.0,!=4.21.0,<5; python_version > '3.9' and sys_platform == 'linux'",
+ "protobuf>=3.19.0,!=4.21.0,<5; sys_platform != 'linux'",
+ "setuptools",
+]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: MIT License",
+ "Natural Language :: English",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3 :: Only",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ "Topic :: System :: Logging",
+ "Topic :: System :: Monitoring",
+]
+
+
+#[project.scripts]
+#wandb = "wandb.cli.cli:cli"
+#wb = "wandb.cli.cli:cli"
+
+[project.urls]
+"Source" = "https://github.com/wandb/wandb"
+"Bug Reports" = "https://github.com/wandb/wandb/issues"
+"Documentation" = "https://docs.wandb.ai/"
+
+[tool.setuptools]
+zip-safe = false
+packages = ["wandb"]
+package-dir = { "wandb" = "wandb" }

core/lang/py/lib/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

core/lang/py/lib/wandb/.gitignore

@@ -0,0 +1,2 @@
+proto/
+lib/

core/lang/py/lib/wandb/__init__.py

@@ -0,0 +1,113 @@
+"""wandb library."""
+
+from wandb.proto import wandb_internal_pb2 as pb2
+
+class _Library:
+ def __init__(self):
+ self._obj = None
+
+ @property
+ def _lib(self):
+ if not self._obj:
+ import ctypes
+ import os
+ import pathlib
+ lib_path = pathlib.Path(__file__).parent / "lib" / "libwandb_core.so"
+ self._obj = ctypes.cdll.LoadLibrary(lib_path)
+ return self._obj
+
+ def new_session(self) -> "Session":
+ return Session(_library=self)
+
+ def teardown(self):
+ pass
+
+
+# global library object
+_library = _Library()
+
+
+class Session:
+ def __init__(self, _library):
+ self._library = _library
+ self._loaded = False
+ self._last_run = None
+
+ @property
+ def _lib(self):
+ return self._library._lib
+
+ def _ensure_loaded(self):
+ if self._loaded:
+ return
+ self._lib.pbSessionSetup()
+ self._loaded = True
+
+ def configure_auth(self):
+ self._ensure_loaded()
+ pass
+
+ def login(self):
+ self._ensure_loaded()
+ pass
+
+ def new_run(self) -> "Run":
+ self._ensure_loaded()
+ run = Run(_session=self)
+ run._start()
+ self._last_run = run
+ return run
+
+ def teardown(self):
+ pass
+
+
+def new_session() -> Session:
+ return _library.new_session()
+
+
+class Run:
+ def __init__(self, _session):
+ self._session = _session
+ self._run_nexus_id = None
+
+ @property
+ def _lib(self):
+ return self._session._lib
+
+ def log(self, data):
+ data_msg = pb2.DataRecord()
+ for k, v in data.items():
+ data_msg.item[k].value_string = "1"
+ data_bytes = data_msg.SerializeToString()
+ # input_buffer = create_string_buffer(data_bytes)
+ self._lib.pbRunLog(self._run_nexus_id, data_bytes, len(data_bytes))
+
+ def _start(self):
+ self._run_nexus_id = self._lib.pbRunStart()
+
+ def finish(self):
+ self._lib.pbRunFinish(self._run_nexus_id)
+
+
+# global default session object
+default_session = new_session()
+
+
+# ---
+# wandb 0.x Compatibility
+# ---
+
+def setup():
+ default_session._ensure_loaded()
+
+
+def init(*args, **kwargs):
+ return default_session.new_run()
+
+def log(*args, **kwargs):
+ default_session._last_run.log(data)
+
+def teardown():
+ global _session
+ _session = None

core/lang/scripts/base-build.sh

@@ -19,10 +19,10 @@ SYSTEM=`uname -s`
 if [ "x$SYSTEM" == "xLinux" ]; then
 CGO_ENABLED=1 go build \
  -ldflags "-extldflags \"-fuse-ld=gold -Wl,--weak-unresolved-symbols\"" \
- -o lang/tmpbuild/embed-core.bin cmd/core/main.go
+ -o lang/tmpbuild/embed-core.bin cmd/wandb-core/main.go
 else
 go build \
- -o lang/tmpbuild/embed-core.bin cmd/core/main.go
+ -o lang/tmpbuild/embed-core.bin cmd/wandb-core/main.go
 fi
 cd -