From 35cb847388e06ba804d0e273ef0a4a70b693356a Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 8 Oct 2023 21:29:50 +0000 Subject: [PATCH] v1.5.0 --- README.md | 11 ++ RELEASE.md | 5 + VERSION | 2 +- .../ClientLib/lib/AMQPListener.py | 78 +++++++++ deployment-examples/ClientLib/lib/MCP.py | 162 ++++++++++++++++++ .../ClientLib/lib/SIODrawer.py | 155 +++++++++++++++++ deployment-examples/README.md | 9 +- .../VideoStreamsConsumer/README.md | 37 ++++ .../clients/python/SIOOutput/Dockerfile | 10 ++ .../clients/python/SIOOutput/SIO.py | 41 +++++ .../python/SIOOutput/docker-compose.yml | 15 ++ .../clients/python/SIOOutput/main.py | 26 +++ .../clients/python/SIOOutput/requirements.txt | 3 + .../config/analytics/boxFilter.json | 20 +++ .../config/analytics/extension.py | 37 ++++ .../config/analytics/extensionConfig1.json | 0 .../config/analytics/extensionConfig2.json | 3 + .../config/analytics/pipelines.json | 48 ++++++ .../config/rabbitmq/definitions.json | 60 +++++++ .../config/rabbitmq}/rabbitmq.conf | 0 .../VideoStreamsConsumer/docker-compose.yml | 63 +++++++ .../VideoStreamsRecorder/README.md | 41 +++++ .../python/SIOOutputProcessor/Dockerfile | 13 ++ .../clients/python/SIOOutputProcessor/SIO.py | 153 +++++++++++++++++ .../SIOOutputProcessor/docker-compose.yml | 15 ++ .../clients/python/SIOOutputProcessor/main.py | 37 ++++ .../SIOOutputProcessor/requirements.txt | 7 + .../config/analytics/boxFilter.json | 20 +++ .../config/analytics/pipelines.json | 54 ++++++ .../VideoStreamsRecorder/config/mcp/mcp.yml | 33 ++++ .../config/rabbitmq/definitions.json | 60 +++++++ .../config/rabbitmq/rabbitmq.conf | 13 ++ .../VideoStreamsRecorder/docker-compose.yml | 86 ++++++++++ docs/media/architecture.png | Bin 0 -> 392721 bytes docs/media/folders.png | Bin 0 -> 151457 bytes docs/media/live555.png | Bin 0 -> 572651 bytes docs/media/services.png | Bin 0 -> 65550 bytes examples/SIORtspOutput/src/RTSPStream.py | 4 + examples/SIORtspOutput/src/main.py | 8 +- examples/lib/AMQPListener.py | 30 +++- live555/conf/custom.env | 0 mcp/conf/custom.env | 0 rabbitmq/conf/custom.env | 0 scripts/sh-services | 10 +- {amqp-stats => services/amqp-stats}/README.md | 0 .../amqp-stats}/conf/default.env | 0 {amqp-stats => services/amqp-stats}/disabled | 0 .../amqp-stats}/docker-compose.yml | 0 {live555 => services/live555}/README.MD | 0 .../live555}/conf/default.env | 2 +- .../live555}/docker-compose.yml | 2 +- {mcp => services/mcp}/README.md | 0 {mcp => services/mcp}/conf/default.env | 0 {mcp => services/mcp}/conf/mcp.yml | 0 {mcp => services/mcp}/depends | 0 {mcp => services/mcp}/docker-compose.yml | 0 {rabbitmq => services/rabbitmq}/README.md | 0 .../rabbitmq}/conf/default.env | 0 .../rabbitmq}/conf/definitions.json | 0 services/rabbitmq/conf/rabbitmq.conf | 13 ++ .../rabbitmq}/docker-compose.yml | 0 {sio => services/sio}/README.md | 0 {sio => services/sio}/conf/default.env | 2 +- .../sio}/conf/plugins/DistanceSensor.py | 0 {sio => services/sio}/conf/sio.json | 0 {sio => services/sio}/config.sh | 0 {sio => services/sio}/depends | 0 .../sio}/docker-compose.interactive.yml | 0 {sio => services/sio}/docker-compose.yml | 0 .../sio}/examples/aqueduct/0099-example.env | 0 .../sio}/examples/aqueduct/README.md | 0 .../sio}/examples/aqueduct/sio.json | 0 .../sio}/examples/camera/sio.json | 0 .../sio}/examples/folderwatch/sio.json | 0 .../sio}/examples/live555/sio.json | 0 .../sio}/examples/plugins/0099-example.env | 0 .../sio}/examples/plugins/sio.json | 0 {sio => services/sio}/sio-shell.sh | 0 sio/conf/custom.env | 0 79 files changed, 1370 insertions(+), 18 deletions(-) create mode 100644 deployment-examples/ClientLib/lib/AMQPListener.py create mode 100644 deployment-examples/ClientLib/lib/MCP.py create mode 100644 deployment-examples/ClientLib/lib/SIODrawer.py create mode 100644 deployment-examples/VideoStreamsConsumer/README.md create mode 100644 deployment-examples/VideoStreamsConsumer/clients/python/SIOOutput/Dockerfile create mode 100644 deployment-examples/VideoStreamsConsumer/clients/python/SIOOutput/SIO.py create mode 100644 deployment-examples/VideoStreamsConsumer/clients/python/SIOOutput/docker-compose.yml create mode 100644 deployment-examples/VideoStreamsConsumer/clients/python/SIOOutput/main.py create mode 100644 deployment-examples/VideoStreamsConsumer/clients/python/SIOOutput/requirements.txt create mode 100644 deployment-examples/VideoStreamsConsumer/config/analytics/boxFilter.json create mode 100644 deployment-examples/VideoStreamsConsumer/config/analytics/extension.py rename amqp-stats/conf/custom.env => deployment-examples/VideoStreamsConsumer/config/analytics/extensionConfig1.json (100%) create mode 100644 deployment-examples/VideoStreamsConsumer/config/analytics/extensionConfig2.json create mode 100644 deployment-examples/VideoStreamsConsumer/config/analytics/pipelines.json create mode 100644 deployment-examples/VideoStreamsConsumer/config/rabbitmq/definitions.json rename {rabbitmq/conf => deployment-examples/VideoStreamsConsumer/config/rabbitmq}/rabbitmq.conf (100%) create mode 100644 deployment-examples/VideoStreamsConsumer/docker-compose.yml create mode 100644 deployment-examples/VideoStreamsRecorder/README.md create mode 100644 deployment-examples/VideoStreamsRecorder/clients/python/SIOOutputProcessor/Dockerfile create mode 100644 deployment-examples/VideoStreamsRecorder/clients/python/SIOOutputProcessor/SIO.py create mode 100644 deployment-examples/VideoStreamsRecorder/clients/python/SIOOutputProcessor/docker-compose.yml create mode 100644 deployment-examples/VideoStreamsRecorder/clients/python/SIOOutputProcessor/main.py create mode 100644 deployment-examples/VideoStreamsRecorder/clients/python/SIOOutputProcessor/requirements.txt create mode 100644 deployment-examples/VideoStreamsRecorder/config/analytics/boxFilter.json create mode 100644 deployment-examples/VideoStreamsRecorder/config/analytics/pipelines.json create mode 100644 deployment-examples/VideoStreamsRecorder/config/mcp/mcp.yml create mode 100644 deployment-examples/VideoStreamsRecorder/config/rabbitmq/definitions.json create mode 100644 deployment-examples/VideoStreamsRecorder/config/rabbitmq/rabbitmq.conf create mode 100644 deployment-examples/VideoStreamsRecorder/docker-compose.yml create mode 100644 docs/media/architecture.png create mode 100644 docs/media/folders.png create mode 100644 docs/media/live555.png create mode 100644 docs/media/services.png delete mode 100644 live555/conf/custom.env delete mode 100644 mcp/conf/custom.env delete mode 100644 rabbitmq/conf/custom.env rename {amqp-stats => services/amqp-stats}/README.md (100%) rename {amqp-stats => services/amqp-stats}/conf/default.env (100%) rename {amqp-stats => services/amqp-stats}/disabled (100%) rename {amqp-stats => services/amqp-stats}/docker-compose.yml (100%) rename {live555 => services/live555}/README.MD (100%) rename {live555 => services/live555}/conf/default.env (79%) rename {live555 => services/live555}/docker-compose.yml (92%) rename {mcp => services/mcp}/README.md (100%) rename {mcp => services/mcp}/conf/default.env (100%) rename {mcp => services/mcp}/conf/mcp.yml (100%) rename {mcp => services/mcp}/depends (100%) rename {mcp => services/mcp}/docker-compose.yml (100%) rename {rabbitmq => services/rabbitmq}/README.md (100%) rename {rabbitmq => services/rabbitmq}/conf/default.env (100%) rename {rabbitmq => services/rabbitmq}/conf/definitions.json (100%) create mode 100644 services/rabbitmq/conf/rabbitmq.conf rename {rabbitmq => services/rabbitmq}/docker-compose.yml (100%) rename {sio => services/sio}/README.md (100%) rename {sio => services/sio}/conf/default.env (93%) rename {sio => services/sio}/conf/plugins/DistanceSensor.py (100%) rename {sio => services/sio}/conf/sio.json (100%) rename {sio => services/sio}/config.sh (100%) rename {sio => services/sio}/depends (100%) rename {sio => services/sio}/docker-compose.interactive.yml (100%) rename {sio => services/sio}/docker-compose.yml (100%) rename {sio => services/sio}/examples/aqueduct/0099-example.env (100%) rename {sio => services/sio}/examples/aqueduct/README.md (100%) rename {sio => services/sio}/examples/aqueduct/sio.json (100%) rename {sio => services/sio}/examples/camera/sio.json (100%) rename {sio => services/sio}/examples/folderwatch/sio.json (100%) rename {sio => services/sio}/examples/live555/sio.json (100%) rename {sio => services/sio}/examples/plugins/0099-example.env (100%) rename {sio => services/sio}/examples/plugins/sio.json (100%) rename {sio => services/sio}/sio-shell.sh (100%) delete mode 100644 sio/conf/custom.env diff --git a/README.md b/README.md index 9b001a1..55f7f5b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This repository hosts a collection of services for the SIO (Sighthound.IO) ecosystem. Services are intended to run via the `./scripts/sh-services` script, which relies on docker-compose. However, docker-compose is not strictly necessary for configuring or using this repository. +![System architecture](docs/media/architecture.png) + The included services are as follow: - SIO: The computer vision analytics engine. - MCP: Media manager service, which includes a REST API and a cleaner. MCP relies on sharing media store folders with SIO service, and listens on AMQP message bus for media creation events, such as new video recording segments or event-driven jpeg images. It then provides the API access to that media (for documentation go to http://localhost:9097), and control its lifecycle. @@ -15,8 +17,15 @@ The `./scripts/sh-services` script is a basic tool that triggers `docker-compose In the turnkey scenario, each service is managed with an individual `docker-compose` configuration file, an optional (or autogenerated by sh-serviecs) `.env` file containing relevant environment variable, and a collection of service specific configuration file in conf subfolder. To assist in orchestrating the services collection and disjointed `docker-compose` and environment configuration, `sh-services` CLI utility was introduced. +![Folder Structure](docs/media/folders.png) + ## Configuration Priority: +First, let's take a look on how the services work: + +![Folder Structure](docs/media/services.png) + + For example, if you have the following configuration files: - default.env - 0009-customer.env @@ -33,6 +42,8 @@ The first file is given the highest priority, so it will overwrite any conflicti This guide will help you set up SIO to point to a fake RTSP generated by live555 and start processing video. +![live555](docs/media/live555.png) + ### Prerequisites (for non-dnncam devices) On Sighthound DNNCam devices, services come preinstalled, and the device GUI interacts with it. If you are using a dnncam we suggest you rely on the GUI to configure/update services, though it's not a requirement. On other devices, you need to manually: diff --git a/RELEASE.md b/RELEASE.md index fe244a6..fb31973 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,4 +1,9 @@ # Release Notes +## v1.5.0 +- services: Migrate services to its own services path +- README.md: Add visual documentation +- examples: Add VideoStreamsConsumer integrated deployment sample + ## v1.4.2 - examples: Fixes issue with pip installation diff --git a/VERSION b/VERSION index c432e90..2e7bd91 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.4.2 +v1.5.0 diff --git a/deployment-examples/ClientLib/lib/AMQPListener.py b/deployment-examples/ClientLib/lib/AMQPListener.py new file mode 100644 index 0000000..650588a --- /dev/null +++ b/deployment-examples/ClientLib/lib/AMQPListener.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +import pika, json +import traceback +import socket + +class AMQPListener: + host = 'localhost' + exchange = 'anypipe' + routing_key = '#' + port = 5672 + + def __init__(self,conf): + self.queue_name = None + self.channel = None + self.connection = None + self.json_callback = None + self.host = conf.get("host", AMQPListener.host) + self.exchange = conf.get("exchange", AMQPListener.exchange) + self.routing_key = conf.get("routing_key", AMQPListener.routing_key) + self.port = conf.get("port", AMQPListener.port) + self.json_callback = lambda data: print(f"Received data {data}") + + def set_callback(self, json_callback): + self.json_callback = json_callback + + def connect(self): + if not self.connection: + self.connection = pika.BlockingConnection( + pika.ConnectionParameters(host=self.host,port=self.port)) + self.channel = self.connection.channel() + + self.channel.exchange_declare(exchange=self.exchange, exchange_type='topic', durable=True) + + def get_queue_name(self): + if not self.queue_name: + self.connect() + result = self.channel.queue_declare(queue='', exclusive=True) + self.queue_name = result.method.queue + print(f"Using queue name {self.queue_name}") + return self.queue_name + + def callback(self,ch, method, properties, body): + try: + data = json.loads(body) + if self.json_callback: + self.json_callback(data) + except Exception as e: + print(f"Caught exception {e} handling callback") + traceback.print_exc() + + def start(self): + """ + Start the amqp listener, setting up a callback at @param json_callback + function with single argument representing a JSON payload. + """ + print(f"Starting AMQP Listener on {self.host}:{self.port}") + try: + self.connect() + except socket.gaierror as e: + traceback.print_exc() + print(f"Error connecting to AMQP host: {self.host}:{self.port}. {e}.") + print("Please check your AMQP configuration. Did you start RabbitMQ?") + print("You can also use the environment variables AMQP_HOST and AMQP_PORT to configure the host and port.") + return + except Exception as e: + traceback.print_exc() + print(f"Caught exception '{e}' connecting to AMQP") + return + queue_name = self.get_queue_name() + self.channel.queue_bind(exchange=self.exchange, queue=queue_name, + routing_key=self.routing_key) + self.channel.basic_consume( + queue=queue_name, on_message_callback=self.callback, auto_ack=True) + print(' [*] Listening for AMQP messages. To exit press CTRL+C') + self.channel.start_consuming() + + def stop(self): + self.channel.stop_consuming() diff --git a/deployment-examples/ClientLib/lib/MCP.py b/deployment-examples/ClientLib/lib/MCP.py new file mode 100644 index 0000000..33672ae --- /dev/null +++ b/deployment-examples/ClientLib/lib/MCP.py @@ -0,0 +1,162 @@ +import requests +from PIL import Image +import numpy as np +from io import BytesIO + +class MCPClient: + def __init__(self, conf): + self.host = conf.get("host", "mcp") + self.port = conf.get("port", 9097) + self.user = conf.get("username", None) + self.password = conf.get("password", None) + if self.user and self.password: + print(f"Connecting to mcp://{self.user}:*****@{self.host}:{self.port}") + else: + print(f"Connecting to mcp://{self.host}:{self.port}") + + def get(self, url): + if self.user and self.password: + auth = (self.user, self.password) + else: + auth = None + response = requests.get(url, auth=auth) + + if response.status_code == 401: + raise Exception("Unauthorized") + else: + return response + + # curl mcp:9097/hlsfs/source + def list_sources(self): + url = f"http://{self.host}:{self.port}/hlsfs/source" + return self.get(url).json() + + # curl mcp:9097/hlsfs/source//stats + def get_stats(self, source_id): + url = f"http://{self.host}:{self.port}/hlsfs/source/{source_id}/stats" + return self.get(url).json() + + # curl mcp:9097/hlsfs/source//image/ + def get_image(self, source_id, image): + url = f"http://{self.host}:{self.port}/hlsfs/source/{source_id}/image/{image}" + response = self.get(url) + + if response.status_code != 200: + if response.status_code == 404: + raise Exception("Image not found") + else: + raise Exception("Error downloading image", response.status_code) + else: + # Convert image to numpy array + img = Image.open(BytesIO(response.content)) + arr = np.array(img) + return arr + + # curl mcp:9097/hlsfs/source//segment/