Source code for AFL.automation.shared.TiledProxyDriver

"""An AFL API server backed by an existing Tiled HTTP service."""

import argparse
import json
import logging
import signal
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urljoin, urlsplit, urlunsplit

from AFL.automation.APIServer.Driver import Driver


[docs] def load_profile(filename): """Read connection settings from a private Tiled client or NAS YAML.""" import yaml profile = yaml.safe_load(Path(filename).read_text()) if not isinstance(profile, dict) or profile.get("trees") is not None: raise ValueError( "The AFL Tiled proxy requires a client profile, " "not a local catalog configuration." ) uri = profile.get("uri") management = profile.get("management", {}) if not uri and management.get("host"): port = profile.get("uvicorn", {}).get("port", 8000) uri = f"http://{management['host']}:{port}" parsed = urlsplit(str(uri or "").strip()) if parsed.scheme not in ("http", "https") or not parsed.netloc: raise ValueError("The Tiled profile must provide an HTTP or HTTPS uri.") key = str( profile.get("api_key") or profile.get("authentication", {}).get("single_user_api_key") or "" ).strip() if not key: authorization = str(profile.get("headers", {}).get("Authorization", "")) if authorization.lower().startswith("apikey "): key = authorization[7:].strip() else: key = authorization.strip() return { "uri": urlunsplit((parsed.scheme, parsed.netloc, "/", "", "")), "api_key": key, "structure_clients": profile.get("structure_clients", "dask"), }
[docs] class TiledProxyDriver(Driver): """Forward Tiled HTTP requests and provide the standard AFL browser. Parameters ---------- uri : str, optional Backend URL. When omitted, use the persistent driver configuration or the value supplied in overrides. api_key : str, optional Backend API key, kept outside the persistent driver configuration. structure_clients : str, optional Tiled structure client selection for the inherited AFL browser. overrides : dict, optional Configuration values that override the driver defaults. http_client : httpx.Client, optional HTTP client supplied by the caller, for example with a mock transport. afl_home : str or pathlib.Path, optional Directory containing the driver's persistent configuration. """ defaults = { "uri": "", "structure_clients": "dask", "timeout": 10.0, }
[docs] def __init__( self, uri=None, api_key="", structure_clients=None, overrides=None, *, http_client=None, afl_home=None, ): """Initialize configuration without opening a backend connection.""" overrides = dict(overrides or {}) if uri is not None: overrides["uri"] = uri if structure_clients is not None: overrides["structure_clients"] = structure_clients super().__init__( name="TiledProxyDriver", defaults=self.gather_defaults(), overrides=overrides, afl_home=afl_home, ) # Credentials belong to the private YAML, not persistent driver config. self._api_key = api_key self._http_client = http_client
[docs] def connect(self): """Create the backend HTTP client on first use.""" if self._http_client is None: import httpx headers = {} if self._api_key: headers["Authorization"] = f"Apikey {self._api_key}" self._http_client = httpx.Client( base_url=self.config["uri"], timeout=self.config["timeout"], headers=headers, follow_redirects=False, )
[docs] def close(self): """Close cached HTTP and Tiled clients; safe to call more than once.""" if self._http_client is not None: self._http_client.close() self._http_client = None if self._tiled_client is not None: self._tiled_client.context.close() self._tiled_client = None
[docs] def status(self): """Return backend information without making a network request.""" return [f"Tiled backend: {self.config['uri']}"]
def _read_tiled_config(self): """Use this backend for the inherited AFL browser routes.""" # The inherited AFL browser reads this connection, independently of # whichever other Tiled service is active in global AFL configuration. return { "status": "success", "tiled_server": self.config["uri"], "tiled_api_key": self._api_key, }
[docs] def upstream_request(self, method, path, query=b"", body=b"", headers=None): """Forward one request using the configured backend credentials.""" self.connect() forwarded_headers = dict(headers or {}) if self._api_key: forwarded_headers["Authorization"] = f"Apikey {self._api_key}" request_path = "/" + path.lstrip("/") if query: request_path += "?" + query.decode("ascii") response = self._http_client.request( method, request_path, content=body, headers=forwarded_headers, ) if (method.upper() == "GET" and request_path.split("?", 1)[0].startswith("/api/v1/distinct/") and response.status_code in (405, 500, 501)): logging.getLogger(__name__).warning( "Tiled distinct returned HTTP %s; collecting values from metadata search", response.status_code, ) return self._distinct_from_search(path, query, forwarded_headers, response.request) return response
def _distinct_from_search(self, path, query, headers, original_request): """Recover native distinct results from every page of a filtered search.""" import httpx params = parse_qsl(query.decode("ascii"), keep_blank_values=True) fields = list(dict.fromkeys(value for key, value in params if key == "metadata")) options = dict(params) def enabled(name): return options.get(name, "false").lower() in ("true", "1", "yes", "on") counts = enabled("counts") structure_families = enabled("structure_families") specs = enabled("specs") groups = {"metadata": {field: {} for field in fields}} if fields else {} if structure_families: groups["structure_families"] = {} if specs: groups["specs"] = {} search_params = [ (key, value) for key, value in params if key not in ("metadata", "counts", "structure_families", "specs", "fields") and not key.startswith("page[") ] search_params.append(("page[limit]", "100")) if fields: search_params.append(("fields", "metadata")) if structure_families: search_params.append(("fields", "structure_family")) if specs: search_params.append(("fields", "specs")) search_path = "/" + path.lstrip("/").replace("api/v1/distinct/", "api/v1/search/", 1) next_url = search_path + "?" + urlencode(search_params) visited = set() def add(group, value): # JSON keys retain the distinction between strings, numbers and # booleans and also support list/dict metadata values. key = json.dumps(value, sort_keys=True, ensure_ascii=False) if key not in group: group[key] = {"value": value, **({"count": 0} if counts else {})} if counts: group[key]["count"] += 1 while next_url: url = self._http_client.base_url.join(next_url) if url in visited: raise ValueError("Tiled metadata search repeated a pagination link") visited.add(url) response = self._http_client.get(url, headers=headers) if not response.is_success: return response payload = response.json() for item in payload["data"]: attributes = item.get("attributes", {}) metadata = attributes.get("metadata", {}) for field in fields: value = metadata for segment in field.split("."): value = value.get(segment) if isinstance(value, dict) else None add(groups["metadata"][field], value) if structure_families: add(groups["structure_families"], attributes.get("structure_family")) if specs: add(groups["specs"], attributes.get("specs", [])) next_url = payload.get("links", {}).get("next") if next_url: resolved = urlsplit(urljoin(str(url), next_url)) upstream = urlsplit(str(self._http_client.base_url)) if ((resolved.scheme, resolved.netloc) != (upstream.scheme, upstream.netloc) or resolved.path != search_path): raise ValueError("Tiled metadata search returned an unexpected pagination link") next_url = urlunsplit(("", "", resolved.path, resolved.query, "")) result = {} if fields: result["metadata"] = {field: list(values.values()) for field, values in groups["metadata"].items()} for name in ("structure_families", "specs"): if name in groups: result[name] = list(groups[name].values()) return httpx.Response(200, json=result, request=original_request)
[docs] def create_server(profile, *, afl_home=None, http_client=None): """Create an APIServer with AFL routes and native Tiled REST forwarding.""" from AFL.automation.APIServer.APIServer import APIServer from flask import Response, jsonify, request driver = TiledProxyDriver( api_key=profile.get("api_key", ""), overrides={ "uri": profile["uri"], "structure_clients": profile.get("structure_clients", "dask"), }, http_client=http_client, afl_home=afl_home, ) api = APIServer("TiledProxy", afl_home=afl_home) api.add_standard_routes() api.create_queue(driver) def proxy(path=""): # Tiled's native REST routes also serve Andon's existing bundled browser. forwarded_headers = { name: request.headers[name] for name in ("Accept", "Content-Type", "Range") if name in request.headers } try: response = driver.upstream_request( method=request.method, path="api/v1/" + path, query=request.query_string, body=request.get_data(), headers=forwarded_headers, ) except Exception: api.app.logger.exception("Could not reach the configured Tiled backend") return jsonify(detail="The configured Tiled backend is unreachable."), 502 content = response.content content_type = response.headers.get("content-type", "application/octet-stream") if "json" in content_type: try: payload = rewrite_links( response.json(), driver.config["uri"], request.url_root ) content = json.dumps(payload).encode() except (ValueError, TypeError): pass response_header_names = ( "content-type", "content-disposition", "content-range", "accept-ranges", "etag", "last-modified", ) headers = { name: value for name, value in response.headers.items() if name.lower() in response_header_names } return Response(content, status=response.status_code, headers=headers) methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] api.app.add_url_rule("/api/v1/", "tiled_api_root", proxy, methods=methods) api.app.add_url_rule("/api/v1/<path:path>", "tiled_api", proxy, methods=methods) @api.app.get("/healthz") def health(): try: response = driver.upstream_request("GET", "api/v1/metadata/") except Exception: return jsonify(status="unavailable"), 502 status = "ready" if response.is_success else "unavailable" return jsonify(status=status), response.status_code return api
[docs] def main(): """Launch the proxy from a private Tiled connection profile.""" from werkzeug.serving import make_server parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8001) args = parser.parse_args() if not 1 <= args.port <= 65535: parser.error("--port must be between 1 and 65535") profile = load_profile(args.config) api = create_server(profile) api.driver.connect() http_server = make_server(args.host, args.port, api.app, threaded=True) # AFL queue_state and all existing browser routes are provided by APIServer. api.init() def stop(_signal, _frame): raise KeyboardInterrupt signal.signal(signal.SIGTERM, stop) print( f"AFL Tiled proxy at http://{args.host}:{args.port}, backend {profile['uri']}", flush=True, ) try: http_server.serve_forever() except KeyboardInterrupt: pass finally: http_server.server_close() api.queue_daemon.stop = True api.driver.close()
if __name__ == "__main__": main()