Merge pull request #1 from andreped/huggingface

.github/workflows/deploy.yml

@@ -0,0 +1,20 @@
+name: Deploy HF
+on:
+  push:
+    branches: [ main ]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://andreped:[email protected]/spaces/andreped/dsa4hf main

Dockerfile

@@ -0,0 +1,29 @@
+FROM docker/compose
+
+# Switch to root user
+#USER root
+
+WORKDIR /code
+
+COPY ./dsa /code/dsa
+
+#CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
+
+#RUN docker run -itd -v /var/run/docker.sock:/var/run/docker.sock -v /code/dsa/:/var/tmp/ docker/compose:1.24.1 -f /var/tmp/docker-compose.yml up -d
+
+RUN ls
+
+#RUN sudo chmod 757 /var/run/docker.sock
+
+# CMD ["/bin/bash"]
+# CMD ["ls"]
+#CMD docker run --privileged=true -it -v /var/run/docker.sock:/var/run/docker.sock -v /code/dsa/:/var/tmp/ docker/compose:1.24.1 -f /var/tmp/dsa/docker-compose.yml up -d
+#CMD ["ls"]
+
+ENTRYPOINT [ "/bin/sh" ]
+
+#RUN docker-compose pull
+
+#CMD ["docker", "run", "-it", "-v", "/var/run/docker.sock:/var/run/docker.sock", ]
+
+#docker run -itd -v /var/run/docker.sock:/var/run/docker.sock -v /root/test/:/var/tmp/ docker/compose:1.24.1 -f /var/tmp/docker-compose.yaml up -d

README.md

@@ -1 +1,36 @@
-# dsa4hf
+---
+title: 'Digital Slide Archive: Platform for working with large microscopy images.'
+colorFrom: indigo
+colorTo: indigo
+sdk: docker
+app_port: 8080
+emoji: 🔬
+pinned: false
+license: mit
+---
+
+# dsa4hf
+
+This project was made to showcase developed plugins for Digital Slide Archive through on Hugging Face spaces.
+
+The project is a work-in-progress. I will make a release when I have it working. Stay tuned!
+
+## Getting started
+
+#### Deployment
+
+When the solution is ready, the website should be accessible on [Hugging Face](https://huggingface.co/spaces/andreped/dsa4hf).
+
+#### Development
+
+```
+docker build -t dsa4hf .
+docker run -it -p 8080:8080 dsa4hf
+```
+
+To go inside docker image and debug, at the bottom of the Dockerfile, add `ENTRYPOINT [ "/bin/sh" ]` before running.
+
+## Credit
+
+I did not develop Digital Slide Archive, only implemented some plugins and showcased deployment on Hugging Face space.
+Credit should be given to the developers of DSA for making such an amazing open software solution!

dsa/.dockerignore

@@ -0,0 +1,3 @@
+assetstore
+db
+logs

dsa/.gitignore

@@ -0,0 +1,4 @@
+assetstore/
+db/
+logs/
+*.local.*

dsa/README.rst

@@ -0,0 +1,146 @@
+========================================
+Digital Slide Archive via Docker Compose
+========================================
+
+This directory contains a complete docker-compose set up for the Digital Slide Archive.
+
+Edit the docker-compose.yml file (or add a docker-compose override file) to add mount points for additional data or for exposing additional ports.
+
+Prerequisites
+-------------
+
+Before using this, you need both Docker and docker-compose.  See the `official installation instructions <https://docs.docker.com/compose/install>`_.
+
+The docker-compose file assumes certain file paths.  This has been tested on Ubuntu 20.04.  It will probably work on other Linux variants.
+
+Get the Digital Slide Archive repository::
+
+    git clone https://github.com/DigitalSlideArchive/digital_slide_archive
+
+Start
+-----
+
+Change to the appropriate directory::
+
+    cd digital_slide_archive/devops/dsa/
+
+To get the most recent built docker images, do::
+
+    docker-compose pull
+
+If you don't pull the images, the main image will be built in preference to pulling.
+
+To start the Digital Slide Archive::
+
+    DSA_USER=$(id -u):$(id -g) docker-compose up
+
+This uses your current user id so that database files, logs, assetstore files, and temporary files are owned by the current user.  If you omit setting ``DSA_USER``, files may be created owned by root.
+
+The girder instance can now be accessed at http://localhost:8080. By default, it creates an ``admin`` user with a password of ``password``. Note that this example does not add any default tasks or sample files.  You can log in with the admin user and use the Slicer CLI Web plugin settings to add default tasks (e.g., ``dsarchive/histomicstk:latest``).
+
+Stop
+----
+
+To stop the Digital Slide Archive::
+
+    docker-compose down -v
+
+The ``-v`` option removes unneeded temporary docker volumes.
+
+Sample Data
+-----------
+
+Sample data can be added after performing ``docker-compose up`` by running::
+
+    python utils/cli_test.py dsarchive/histomicstk:latest --test
+
+This downloads the HistomicsTK analysis tools, some sample data, and runs nuclei detection on some of the sample data.  You need Python 3.6 or later available and may need to ``pip install girder-client`` before you can run this command.
+
+
+Development
+-----------
+
+You can log into the running ``girder`` or ``worker`` containers by typing::
+
+    docker-compose exec girder bash
+
+There are two convenience scripts ``restart_girder.sh`` and ``rebuild_and_restart_girder.sh`` that can be run in the container.
+
+You can develop source code by mounting the source directory into the container.  See the ``docker-compose.yml`` file for details.
+
+If you need to log into the container as the Girder user, type::
+
+    docker-compose exec --user $(id -u) girder bash
+
+Technical Details
+-----------------
+
+The Digital Slider Archive is built in Girder and Girder Worker.  Here, these are coordinated using docker-compose.  There are five containers that are started:
+
+- `Girder <https://girder.readthedocs.io/>`_.  Girder is an asset and user management system.  It handles permissions and serves data via http.
+
+- `MongoDB <https://www.mongodb.com/>`_.  Girder stores settings and information about users and assets in a MongoDB database.
+
+- `Girder Worker <https://girder-worker.readthedocs.io/>`_.  Girder Worker is a task runner based on `Celery <https://celery.readthedocs.io/>`_ that has specific features to get authenticated data from Girder.
+
+- `RabbitMQ <https://www.rabbitmq.com/>`_.  Girder communicates to Girder Worker through a broker.  In this configuration it is RabbitMQ.  Girder Worker can be run on multiple computers communicating with a single broker to distribute processing.
+
+- `Memcached <https://memcached.org/>`_.  Memcached is used to cache data for faster access.  This is used for large tiled images.
+
+The Digital Slide Archive relies on several Girder plugins:
+
+- `large_image <https://github.com/girder/large_image>`_.  This provides a standardized way to access a wide range of image formats.  Images can be handled as multi-resolution tiles.  large_image has numerous tile sources to handle different formats.
+
+- `HistomicUI <https://github.com/DigitalSlideArchive/HistomicsUI>`_.  This provides a user interface to examine and annotate large images.
+
+- `Slicer CLI Web <https://github.com/girder/slicer_cli_web>`_.  This can run processing tasks in Docker containers.  Tasks report their capabilities via the Slicer CLI standard, listing required and optional inputs and outputs.  These tasks can be selected and configured via Girder and HistomicsUI and then run in a distributed fashion via Girder Worker.
+
+Slicer CLI Web runs tasks in Docker containers and is itself running in a Docker container (in Girder for determining options and Girder Worker to run the task).  In order to allow a process in a docker container to create another docker container, the paths the docker executable and communications sockets are mounted from the host to the docker container.
+
+Permissions
+-----------
+
+By default, the girder container is run in Docker privileged mode.  This can be reduced to a small set of permissions (see the docker-compose.yml file for details), but these may vary depending on the host system.  If no extra permissions are granted, or if the docker daemon is started with --no-new-privileges, or if libfuse is not installed on the host system, the internal fuse mount will not be started.  This may prevent full functionality with non-filesystem assestores and with some multiple-file image formats.
+
+Customizing
+-----------
+
+Since this uses standard docker-compose, you can customize the process by creating a ``docker-compose.override.yml`` file in the same directory (or a yaml file of any name and use appropriate ``docker-compose -f docker-compose.yml -f <my yaml file> <command>`` command).  Further, if you mount a provisioning yaml file into the docker image, you can customize settings, plugins, resources, and other options.
+
+See the ``docker-compose.yml`` and ``provision.yaml`` files for details.
+
+Example
+~~~~~~~
+
+To add some additional girder plugins and mount additional directories for assetstores, you can do something like this:
+
+``docker-compose.override.yml``::
+
+    ---
+    version: '3'
+    services:
+      girder:
+        environment:
+          # Specify that we want to use the provisioning file
+          DSA_PROVISION_YAML: ${DSA_PROVISION_YAML:-/opt/digital_slide_archive/devops/dsa/provision.yaml}
+        volumes:
+          # Mount the local provisioning file into the container
+          - ./provision.local.yaml:/opt/digital_slide_archive/devops/dsa/provision.yaml
+          # Also expose a local data mount into the container
+          - /mnt/data:/mnt/data
+
+``provision.local.yaml``::
+
+    ---
+    # Load some sample data
+    samples: True
+    # A list of additional pip modules to install
+    pip:
+      - girder-oauth
+      - girder-ldap
+    # rebuild the girder web client since we install some additional plugins
+    rebuild-client: True
+    # List slicer-cli-images to pull and load
+    slicer-cli-image:
+      - dsarchive/histomicstk:latest
+      - girder/slicer_cli_web:small

dsa/docker-compose.yml

@@ -0,0 +1,168 @@
+---
+version: '3'
+services:
+  girder:
+    image: dsarchive/dsa_common
+    build: ../..
+    # Instead of privileged mode, fuse can use:
+    # devices:
+    #   - /dev/fuse:/dev/fuse
+    # security_opt:
+    #   - apparmor:unconfined
+    # cap_add:
+    #   - SYS_ADMIN
+    # but these may be somewhat host specific, so we default to privileged.  If
+    # the docker daemon is being run with --no-new-privileges, fuse may not
+    # work.
+    # See also https://github.com/docker/for-linux/issues/321 for possible
+    # methods to avoid both privileged mode and cap_add SYS_ADMIN.
+    privileged: true
+    # Set DSA_USER to a user id that is part of the docker group (e.g.,
+    # `DSA_USER=$(id -u):$(id -g)`).  This makes files in assetstores and logs
+    # owned by that user and provides permissions to manage docker
+    environment:
+      DSA_USER: ${DSA_USER:-}
+      DSA_PROVISION_YAML: ${DSA_PROVISION_YAML:-/opt/digital_slide_archive/devops/dsa/provision.yaml}
+    restart: unless-stopped
+    # Set DSA_PORT to expose the interface on another port (default 8080).
+    ports:
+      - "${DSA_PORT:-8080}:8080"
+    volumes:
+      # Needed to use slicer_cli_web to run docker containers
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Default assetstore
+      - ./assetstore:/assetstore
+      # Location of girder.cfg
+      - ./girder.cfg:/etc/girder.cfg
+      # Location of provision.py
+      - ./provision.py:/opt/digital_slide_archive/devops/dsa/provision.py
+      - ./provision.yaml:/opt/digital_slide_archive/devops/dsa/provision.yaml
+      - ./start_girder.sh:/opt/digital_slide_archive/devops/dsa/start_girder.sh
+      # Location to store logs
+      - ./logs:/logs
+
+      # For local development, uncomment the set of mounts associated with the
+      # local source files.  Adding the editable egg directories first allows
+      # allow mounting source files from the host without breaking the internal
+      # data.
+
+      # - /opt/girder/girder.egg-info
+      # - /opt/girder/clients/python/girder_client.egg-info
+      # - ../../../girder:/opt/girder
+
+      # - /opt/girder_worker/girder_worker.egg-info
+      # - ../../../../girder_worker:/opt/girder_worker
+
+      # - /opt/girder_worker_utils/girder_worker_utils.egg-info
+      # - ../../../../girder_worker_utils:/opt/girder_worker_utils
+
+      # - /opt/HistomicsUI/histomicsui.egg-info
+      # - ../../../HistomicsUI:/opt/HistomicsUI
+
+      # - /opt/slicer_cli_web/girder_slicer_cli_web.egg-info
+      # - ../../../slicer_cli_web:/opt/slicer_cli_web
+
+      # - /opt/large_image/girder_annotation/girder_large_image_annotation.egg-info
+      # - /opt/large_image/girder/girder_large_image.egg-info
+      # - /opt/large_image/sources/bioformats/large_image_source_bioformats.egg-info
+      # - /opt/large_image/sources/openslide/large_image_source_openslide.egg-info
+      # - /opt/large_image/sources/ometiff/large_image_source_ometiff.egg-info
+      # - /opt/large_image/sources/pil/large_image_source_pil.egg-info
+      # - /opt/large_image/sources/test/large_image_source_test.egg-info
+      # - /opt/large_image/sources/dummy/large_image_source_dummy.egg-info
+      # - /opt/large_image/sources/tiff/large_image_source_tiff.egg-info
+      # - /opt/large_image/sources/mapnik/large_image_source_mapnik.egg-info
+      # - /opt/large_image/sources/openjpeg/large_image_source_openjpeg.egg-info
+      # - /opt/large_image/sources/gdal/large_image_source_gdal.egg-info
+      # - /opt/large_image/sources/nd2/large_image_source_nd2.egg-info
+      # - /opt/large_image/large_image.egg-info
+      # - /opt/large_image/utilities/converter/large_image_converter.egg-info
+      # - /opt/large_image/utilities/tasks/large_image_tasks.egg-info
+      # - ../../../large_image:/opt/large_image
+
+      # Add additional mounts here to get access to existing files on your
+      # system.  Also add them to the worker container to reduce copying.
+    depends_on:
+      - mongodb
+      - memcached
+      - rabbitmq
+    command: /opt/digital_slide_archive/devops/dsa/start_girder.sh
+  mongodb:
+    image: "mongo:latest"
+    # Set DSA_USER to your user id (e.g., `DSA_USER=$(id -u):$(id -g)`)
+    # so that database files are owned by yourself.
+    user: ${DSA_USER:-PLEASE SET DSA_USER}
+    restart: unless-stopped
+    # Limiting maxConns reduces the amount of shared memory demanded by
+    # mongo.  Remove this limit or increase the host vm.max_map_count value.
+    command: --maxConns 1000
+    volumes:
+      # Location to store database files
+      - ./db:/data/db
+    # Uncomment to allow access to the database from outside of the docker
+    # network.
+    # ports:
+    #   - "27017"
+    logging:
+      options:
+        max-size: "10M"
+        max-file: "5"
+  memcached:
+    image: memcached
+    command: -m 4096 --max-item-size 8M
+    restart: unless-stopped
+    # Uncomment to allow access to memcached from outside of the docker network
+    # ports:
+    #   - "11211"
+    logging:
+      options:
+        max-size: "10M"
+        max-file: "5"
+  rabbitmq:
+    image: "rabbitmq:latest"
+    restart: unless-stopped
+    # Uncomment to allow access to rabbitmq from outside of the docker network
+    # ports:
+    #   - "5672"
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER:-}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS:-}
+    volumes:
+      - ./rabbitmq.advanced.config:/etc/rabbitmq/advanced.config:ro
+    logging:
+      options:
+        max-size: "10M"
+        max-file: "5"
+  worker:
+    image: dsarchive/dsa_common
+    build: ../..
+    # Set DSA_USER to a user id that is part of the docker group (e.g.,
+    # `DSA_USER=$(id -u):$(id -g)`).  This provides permissions to manage
+    # docker
+    environment:
+      DSA_USER: ${DSA_USER:-}
+      DSA_WORKER_CONCURRENCY: ${DSA_WORKER_CONCURRENCY:-2}
+      DSA_PROVISION_YAML: ${DSA_PROVISION_YAML:-/opt/digital_slide_archive/devops/dsa/provision.yaml}
+      TMPDIR:
+    restart: unless-stopped
+    volumes:
+      # Needed to use slicer_cli_web to run docker containers
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Modify the worker.local.cfg to specify a different rabbitmq server and
+      # then enable this mount.  On the rabbitmq server, make sure you add a
+      # non-guest default user and use that both in the worker and in the main
+      # girder settings.
+      # - ./worker.local.cfg:/opt/girder_worker/girder_worker/worker.local.cfg
+      # Allow overriding the start command
+      - ./start_worker.sh:/opt/digital_slide_archive/devops/dsa/start_worker.sh
+      # Needed to allow transferring data to slicer_cli_web docker containers
+      - ${TMPDIR:-/tmp}:${TMPDIR:-/tmp}
+      # Add additional mounts here to get access to existing files on your
+      # system if they have the same path as on the girder container.
+    depends_on:
+      - rabbitmq
+    command: /opt/digital_slide_archive/devops/dsa/start_worker.sh
+    logging:
+      options:
+        max-size: "10M"
+        max-file: "5"

dsa/girder.cfg

@@ -0,0 +1,45 @@
+[global]
+server.socket_host = "0.0.0.0"
+server.max_request_body_size = 1073741824
+
+[database]
+uri = "mongodb://mongodb:27017/girder?socketTimeoutMS=3600000"
+
+[server]
+# Set to "production" or "development"
+mode = "development"
+# Disable the event daemon if you do not wish to run event handlers in a background thread.
+# This may be necessary in certain deployment modes.
+disable_event_daemon = False
+
+[logging]
+log_root = "/logs"
+log_access = ["screen", "info"]
+# Log everything to the info log (errors also go to the error log)
+log_max_info_level = "CRITICAL"
+# Increase maximum size of log file
+log_max_size = "10 Mb"
+
+[large_image]
+#  cache_backend is either "memcached" (default) or "python"
+cache_backend = "memcached"
+cache_memcached_url = "memcached"
+cache_memcached_username = None
+cache_memcached_password = None
+#  cache_python_memory_portion affects memory use when using python caching.
+#  Higher numbers use less memory.
+# cache_python_memory_portion = 8
+#  These can be used to reduce the amount of memory used for caching tile
+#  sources
+# cache_tilesource_memory_portion = 16
+cache_tilesource_maximum = 64
+
+[cache]
+enabled = True
+
+[histomicsui]
+# If restrict_downloads is True, only logged-in users can access download
+# and tiles/images endpoints.  If this is a number, file and item download
+# endpoints can be used by anonymous users for files up to the specified
+# size in bytes.  This setting does not affect logged-in users.
+restrict_downloads = 100000

dsa/provision.py

@@ -0,0 +1,649 @@
+#!/usr/bin/env python3
+
+import argparse
+import configparser
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+import time
+
+import yaml
+
+logger = logging.getLogger(__name__)
+# See http://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+
+def get_collection_folder(adminUser, collName, folderName):
+    from girder.models.collection import Collection
+    from girder.models.folder import Folder
+
+    if Collection().findOne({'lowerName': collName.lower()}) is None:
+        logger.info('Create collection %s', collName)
+        Collection().createCollection(collName, adminUser)
+    collection = Collection().findOne({'lowerName': collName.lower()})
+    if Folder().findOne({
+            'parentId': collection['_id'], 'lowerName': folderName.lower()}) is None:
+        logger.info('Create folder %s in %s', folderName, collName)
+        Folder().createFolder(collection, folderName, parentType='collection',
+                              public=True, creator=adminUser)
+    folder = Folder().findOne({'parentId': collection['_id'], 'lowerName': folderName.lower()})
+    return folder
+
+
+def get_sample_data(adminUser, collName='Sample Images', folderName='Images'):
+    """
+    As needed, download sample data.
+
+    :param adminUser: a user to create and modify collections and folders.
+    :param collName: the collection name where the data will be added.
+    :param folderName: the folder name where the data will be added.
+    :returns: the folder where the sample data is located.
+    """
+    try:
+        import girder_client
+    except ImportError:
+        logger.error('girder_client is unavailable.  Cannot get sample data.')
+        return
+    from girder.models.item import Item
+    from girder.models.upload import Upload
+    from girder_large_image.models.image_item import ImageItem
+
+    folder = get_collection_folder(adminUser, collName, folderName)
+    remote = girder_client.GirderClient(apiUrl='https://data.kitware.com/api/v1')
+    remoteFolder = remote.resourceLookup('/collection/HistomicsTK/Deployment test images')
+    sampleItems = []
+    for remoteItem in remote.listItem(remoteFolder['_id']):
+        item = Item().findOne({'folderId': folder['_id'], 'name': remoteItem['name']})
+        if item and len(list(Item().childFiles(item, limit=1))):
+            sampleItems.append(item)
+            continue
+        if not item:
+            item = Item().createItem(remoteItem['name'], creator=adminUser, folder=folder)
+        for remoteFile in remote.listFile(remoteItem['_id']):
+            with tempfile.NamedTemporaryFile() as tf:
+                fileName = tf.name
+                tf.close()
+                logger.info('Downloading %s', remoteFile['name'])
+                remote.downloadFile(remoteFile['_id'], fileName)
+                Upload().uploadFromFile(
+                    open(fileName, 'rb'), os.path.getsize(fileName),
+                    name=remoteItem['name'], parentType='item',
+                    parent=item, user=adminUser)
+        sampleItems.append(item)
+    for item in sampleItems:
+        if 'largeImage' not in item:
+            logger.info('Making large_item %s', item['name'])
+            try:
+                ImageItem().createImageItem(item, createJob=False)
+            except Exception:
+                pass
+            logger.info('done')
+    return folder
+
+
+def value_from_resource(value, adminUser):
+    """
+    If a value is a string that startwith 'resource:', it is a path to an
+    existing resource.  Fetch it an return the string of the _id.
+
+    :param value: a value
+    :returns: the original value it is not a resource, or the string id of the
+        resource.
+    """
+    import girder.utility.path as path_util
+
+    if str(value) == 'resourceid:admin':
+        value = str(adminUser['_id'])
+    elif str(value).startswith('resourceid:'):
+        resource = path_util.lookUpPath(value.split(':', 1)[1], force=True)['document']
+        value = str(resource['_id'])
+    elif str(value) == 'resource:admin':
+        value = adminUser
+    elif str(value).startswith('resource:'):
+        value = path_util.lookUpPath(value.split(':', 1)[1], force=True)['document']
+    return value
+
+
+def provision_resources(resources, adminUser):
+    """
+    Given a dictionary of resources, add them to the system.  The resource is
+    only added if a resource of that name with the same parent object does not
+    exist.
+
+    :param resources: a list of resources to add.
+    :param adminUser: the admin user to use for provisioning.
+    """
+    from girder.utility.model_importer import ModelImporter
+
+    for entry in resources:
+        entry = {k: value_from_resource(v, adminUser) for k, v in entry.items()}
+        modelName = entry.pop('model')
+        metadata = entry.pop('metadata', None)
+        metadata_update = entry.pop('metadata_update', True)
+        metadata_key = entry.pop('metadata_key', 'meta')
+        model = ModelImporter.model(modelName)
+        key = 'name' if model != 'user' else 'login'
+        query = {}
+        if key in entry:
+            query[key] = entry[key]
+        owners = {'folder': 'parent', 'item': 'folder', 'file': 'item'}
+        ownerKey = owners.get(modelName)
+        if ownerKey and ownerKey in entry and isinstance(
+                entry[ownerKey], dict) and '_id' in entry[ownerKey]:
+            query[ownerKey + 'Id'] = entry[ownerKey]['_id']
+        if query and model.findOne(query):
+            result = model.findOne(query)
+            logger.debug('Has %s (%r)', modelName, entry)
+        else:
+            createFunc = getattr(model, 'create%s' % modelName.capitalize())
+            logger.info('Creating %s (%r)', modelName, entry)
+            result = createFunc(**entry)
+        if isinstance(metadata, dict) and hasattr(model, 'setMetadata'):
+            if metadata_key not in metadata or metadata_update:
+                if metadata_key not in result:
+                    result[metadata_key] = {}
+                result[metadata_key].update(metadata.items())
+                for key in metadata:
+                    if metadata[key] is None:
+                        del result[metadata_key][key]
+                model.validateKeys(result[metadata_key])
+                result = model.save(result)
+
+
+def get_slicer_images(imageList, adminUser):
+    """
+    Load a list of cli docker images into the system.
+
+    :param imageList: a list of docker images.
+    :param adminUser: an admin user for permissions.
+    """
+    import threading
+
+    from girder import logger
+    from girder.models.setting import Setting
+    from girder_jobs.constants import JobStatus
+    from girder_jobs.models.job import Job
+    from slicer_cli_web.config import PluginSettings
+    from slicer_cli_web.docker_resource import DockerResource
+    from slicer_cli_web.image_job import jobPullAndLoad
+
+    imageList = [entry for entry in imageList if entry and len(entry)]
+    if not len(imageList):
+        return
+    logger.info('Pulling and installing slicer_cli images: %r', imageList)
+    job = Job().createLocalJob(
+        module='slicer_cli_web.image_job',
+        function='jobPullAndLoad',
+        kwargs={
+            'nameList': imageList,
+            'folder': Setting().get(PluginSettings.SLICER_CLI_WEB_TASK_FOLDER),
+        },
+        title='Pulling and caching docker images',
+        type=DockerResource.jobType,
+        user=adminUser,
+        public=True,
+        asynchronous=True
+    )
+    job = Job().save(job)
+    t = threading.Thread(target=jobPullAndLoad, args=(job, ))
+    t.start()
+    logpos = 0
+    logger.info('Result:\n')
+    while job['status'] not in {JobStatus.SUCCESS, JobStatus.ERROR, JobStatus.CANCELED}:
+        time.sleep(0.1)
+        job = Job().load(id=job['_id'], user=adminUser, includeLog=True)
+        if 'log' in job:
+            while logpos < len(job['log']):
+                logger.info(job['log'][logpos].rstrip())
+                logpos += 1
+    t.join()
+    if 'log' not in job:
+        logger.warning('Job record: %r', job)
+    if job['status'] != JobStatus.SUCCESS:
+        raise Exception('Failed to pull and load images')
+
+
+def preprovision(opts):
+    """
+    Preprovision the instance.  This includes installing python modules with
+    pip and rebuilding the girder client if desired.
+
+    :param opts: the argparse options.
+    """
+    if getattr(opts, 'pip', None) and len(opts.pip):
+        for entry in opts.pip:
+            cmd = 'pip install %s' % entry
+            logger.info('Installing: %s', cmd)
+            subprocess.check_call(cmd, shell=True)
+    if getattr(opts, 'shell', None) and len(opts.shell):
+        for entry in opts.shell:
+            cmd = entry
+            logger.info('Running: %s', cmd)
+            subprocess.check_call(cmd, shell=True)
+    if getattr(opts, 'rebuild-client', None):
+        cmd = 'girder build'
+        if str(getattr(opts, 'rebuild-client', None)).lower().startswith('dev'):
+            cmd += ' --dev'
+        logger.info('Rebuilding girder client: %s', cmd)
+        cmd = ('NPM_CONFIG_FUND=false NPM_CONFIG_AUDIT=false '
+               'NPM_CONFIG_AUDIT_LEVEL=high NPM_CONFIG_LOGLEVEL=error '
+               'NPM_CONFIG_PROGRESS=false NPM_CONFIG_PREFER_OFFLINE=true ' + cmd)
+        subprocess.check_call(cmd, shell=True)
+
+
+def provision(opts):  # noqa
+    """
+    Provision the instance.
+
+    :param opts: the argparse options.
+    """
+    from girder.models.assetstore import Assetstore
+    from girder.models.setting import Setting
+    from girder.models.user import User
+
+    # If there is are no admin users, create an admin user
+    if User().findOne({'admin': True}) is None:
+        adminParams = dict({
+            'login': 'admin',
+            'password': 'password',
+            'firstName': 'Admin',
+            'lastName': 'Admin',
+            'email': '[email protected]',
+            'public': True,
+        }, **(opts.admin if opts.admin else {}))
+        User().createUser(admin=True, **adminParams)
+    adminUser = User().findOne({'admin': True})
+
+    # Make sure we have an assetstore
+    assetstoreParams = opts.assetstore or {'name': 'Assetstore', 'root': '/assetstore'}
+    if not isinstance(assetstoreParams, list):
+        assetstoreParams = [assetstoreParams]
+    if Assetstore().findOne() is None:
+        for params in assetstoreParams:
+            method = params.pop('method', 'createFilesystemAssetstore')
+            getattr(Assetstore(), method)(**params)
+
+    # Make sure we have a demo collection and download some demo files
+    if getattr(opts, 'samples', None):
+        get_sample_data(
+            adminUser,
+            getattr(opts, 'sample-collection', 'Samples'),
+            getattr(opts, 'sample-folder', 'Images'))
+    if opts.resources:
+        provision_resources(opts.resources, adminUser)
+    settings = dict({}, **(opts.settings or {}))
+    force = getattr(opts, 'force', None) or []
+    for key, value in settings.items():
+        if (value != '__SKIP__' and (
+                force is True or key in force or
+                Setting().get(key) is None or
+                Setting().get(key) == Setting().getDefault(key))):
+            value = value_from_resource(value, adminUser)
+            logger.info('Setting %s to %r', key, value)
+            Setting().set(key, value)
+    if getattr(opts, 'slicer-cli-image', None):
+        try:
+            get_slicer_images(getattr(opts, 'slicer-cli-image', None), adminUser)
+        except Exception:
+            logger.info('Cannot fetch slicer-cli-images.')
+
+
+def preprovision_worker(opts):
+    """
+    Preprovision the worker.
+    """
+    settings = dict({}, **(opts.worker or {}))
+    if settings.get('pip') and len(settings['pip']):
+        for entry in settings['pip']:
+            cmd = 'pip install %s' % entry
+            logger.info('Installing: %s', cmd)
+            subprocess.check_call(cmd, shell=True)
+    if settings.get('shell') and len(settings['shell']):
+        for entry in settings['shell']:
+            cmd = entry
+            logger.info('Running: %s', cmd)
+            subprocess.check_call(cmd, shell=True)
+
+
+def provision_worker(opts):
+    """
+    Provision the worker.  There are a few top-level settings, but others
+    should be in the worker sub-field.
+    """
+    settings = dict({}, **(opts.worker or {}))
+    for key in dir(opts):
+        if key.startswith('worker-'):
+            mainkey = key.split('worker-', 1)[1]
+            if settings.get(mainkey) is None:
+                settings[mainkey] = getattr(opts, key)
+    if not settings.get('rabbitmq-host'):
+        return
+    conf = configparser.ConfigParser()
+    conf.read([settings['config']])
+    conf.set('celery', 'broker', 'amqp://%s:%s@%s/' % (
+        settings['rabbitmq-user'], settings['rabbitmq-pass'], settings['host']))
+    conf.set('celery', 'backend', 'rpc://%s:%s@%s/' % (
+        settings['rabbitmq-user'], settings['rabbitmq-pass'], settings['host']))
+    with open(settings['config'], 'w') as fptr:
+        conf.write(fptr)
+
+
+def merge_environ_opts(opts):
+    """
+    Merge environment options, overriding other settings.
+
+    :param opts: the options parsed from the command line.
+    :return opts: the modified options.
+    """
+    keyDict = {
+        'RABBITMQ_USER': 'worker_rabbitmq_user',
+        'RABBITMQ_PASS': 'worker_rabbitmq_pass',
+        'DSA_RABBITMQ_HOST': 'worker_rabbitmq_host',
+    }
+    for key, value in os.environ.items():
+        if not value or not value.strip():
+            continue
+        if key == 'DSA_WORKER_API_URL':
+            key = 'worker.api_url'
+        elif key.startswith('DSA_SETTING_'):
+            key = key.split('DSA_SETTING_', 1)[1]
+        elif key in keyDict:
+            key = keyDict[key]
+        else:
+            continue
+        opts.settings[key] = value
+        if not opts.force:
+            opts.force = {key}
+        elif opts.force is not True:
+            opts.force.add(key)
+    return opts
+
+
+def merge_yaml_opts(opts, parser):
+    """
+    Parse a yaml file of provisioning options.  Modify the options used for
+    provisioning.
+
+    :param opts: the options parsed from the command line.
+    :param parser: command line parser used to check if the options are the
+        default values.
+    :return opts: the modified options.
+    """
+    yamlfile = os.environ.get('DSA_PROVISION_YAML') if getattr(
+        opts, 'yaml', None) is None else opts.yaml
+    if yamlfile:
+        logger.debug('Parse yaml file: %r', yamlfile)
+    if not yamlfile or not os.path.exists(yamlfile):
+        return opts
+    defaults = parser.parse_args(args=[])
+    if getattr(opts, 'use-defaults', None) is not False:
+        defaults = merge_default_opts(defaults)
+    yamlopts = yaml.safe_load(open(yamlfile).read())
+    for key, value in yamlopts.items():
+        key = key.replace('_', '-')
+        if getattr(opts, key, None) is None or getattr(
+                opts, key, None) == getattr(defaults, key, None):
+            if key == 'settings' and getattr(opts, key, None) and isinstance(value, dict):
+                getattr(opts, key).update(value)
+            else:
+                setattr(opts, key, value)
+    logger.debug('Arguments after adding yaml: %r', opts)
+    return opts
+
+
+def merge_default_opts(opts):
+    """
+    Add the defaults to the options.
+
+    :param opts: the options parsed from the command line.
+    :return opts: the modified options.
+    """
+    settings = dict({}, **(opts.settings or {}))
+    settings.update({
+        'worker.broker': 'amqp://guest:guest@rabbitmq',
+        'worker.backend': 'rpc://guest:guest@rabbitmq',
+        'worker.api_url': 'http://girder:8080/api/v1',
+        'worker.direct_path': True,
+        'core.brand_name': 'Digital Slide Archive',
+        'histomicsui.webroot_path': 'histomics',
+        'histomicsui.alternate_webroot_path': 'histomicstk',
+        'histomicsui.delete_annotations_after_ingest': True,
+        'homepage.markdown': """# Digital Slide Archive
+---
+## Bioinformatics Platform
+
+Welcome to the **Digital Slide Archive**.
+
+Developers who want to use the Girder REST API should check out the
+[interactive web API docs](api/v1).
+
+The [HistomicsUI](histomics) application is enabled.""",
+        'slicer_cli_web.task_folder': 'resourceid:collection/Tasks/Slicer CLI Web Tasks',
+    })
+    opts.settings = settings
+    if getattr(opts, 'slicer-cli-image', None) is None:
+        setattr(opts, 'slicer-cli-image', ['dsarchive/histomicstk:latest'])
+    if getattr(opts, 'assetstore', None) is None:
+        opts.assetstore = {
+            'name': 'Assetstore',
+            'root': '/assetstore',
+            'method': 'createFilesystemAssetstore',
+        }
+    if getattr(opts, 'admin', None) is None:
+        opts.admin = {
+            'login': 'admin',
+            'password': 'password',
+            'firstName': 'Admin',
+            'lastName': 'Admin',
+            'email': '[email protected]',
+            'public': True,
+        }
+    resources = opts.resources or []
+    resources.extend([{
+        'model': 'collection',
+        'name': 'Tasks',
+        'creator': 'resource:admin',
+        'public': True,
+    }, {
+        'model': 'folder',
+        'parent': 'resource:collection/Tasks',
+        'parentType': 'collection',
+        'name': 'Slicer CLI Web Tasks',
+        'creator': 'resource:admin',
+        'public': True,
+    }])
+    opts.resources = resources
+    return opts
+
+
+class YamlAction(argparse.Action):
+    def __init__(self, option_strings, dest, nargs=None, **kwargs):
+        """Parse a yaml entry"""
+        if nargs is not None:
+            raise ValueError('nargs not allowed')
+        super().__init__(option_strings, dest, **kwargs)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, yaml.safe_load(values))
+
+
+if __name__ == '__main__':  # noqa
+    parser = argparse.ArgumentParser(description='Provision a Digital Slide Archive instance')
+    parser.add_argument(
+        '--force', action='store_true',
+        help='Reset all settings.  This does not change the admin user or the '
+        'default assetstore if those already exist.  Otherwise, settings are '
+        'only added or modified if they do not exist or are the default '
+        'value.')
+    parser.add_argument(
+        '--samples', '--data', '--sample-data',
+        action='store_true', help='Download sample data')
+    parser.add_argument(
+        '--sample-collection', dest='sample-collection', default='Samples',
+        help='Sample data collection name')
+    parser.add_argument(
+        '--sample-folder', dest='sample-folder', default='Images',
+        help='Sample data folder name')
+    parser.add_argument(
+        '--admin', action=YamlAction,
+        help='A yaml dictionary of parameters used to create a default admin '
+        'user.  If any of login, password, firstName, lastName, email, or '
+        'public are not specified, some default values are used.  If any '
+        'admin user already exists, no modifications are made.')
+    parser.add_argument(
+        '--assetstore', action=YamlAction,
+        help='A yaml dictionary (or list of dictionaries) of parameters used '
+        'to create a default assetstore.  This can include "method" which '
+        'includes the creation method, such as "createFilesystemAssetstore" '
+        'or "createS3Assetstore".  Otherwise, this is a list of parameters '
+        'passed to the creation method.  For filesystem assetstores, these '
+        'parameters are name, root, and perms.  For S3 assetstores, these are '
+        'name, bucket, accessKeyId, secret, prefix, service, readOnly, '
+        'region, inferCredentials, and serverSideEncryption.  If unspecified, '
+        'a filesystem assetstore is created.')
+    parser.add_argument(
+        '--settings', action=YamlAction,
+        help='A yaml dictionary of settings to change in the Girder '
+        'database.  This is merged with the default settings dictionary.  '
+        'Settings are only changed if they are their default values, the '
+        'force option is used, or they are specified by an environment '
+        'variable.  If a setting has a value of "__SKIP__", it will not be '
+        'changed (this can prevent setting a default setting '
+        'option to any value).')
+    parser.add_argument(
+        '--resources', action=YamlAction,
+        help='A yaml list of resources to add by name to the Girder '
+        'database.  Each entry is a dictionary including "model" with the '
+        'resource model and a dictionary of values to pass to the '
+        'appropriate create(resource) function.  A value of '
+        '"resource:<path>" is converted to the resource document with that '
+        'resource path.  "resource:admin" uses the default admin, '
+        '"resourceid:<path>" is the string id for the resource path.')
+    parser.add_argument(
+        '--yaml',
+        help='Specify parameters for this script in a yaml file.  If no value '
+        'is specified, this defaults to the environment variable of '
+        'DSA_PROVISION_YAML.  No error is thrown if the file does not exist. '
+        'The yaml file is a dictionary of keys as would be passed to the '
+        'command line.')
+    parser.add_argument(
+        '--no-mongo-compat', action='store_false', dest='mongo-compat',
+        default=True, help='Do not automatically set the mongo feature '
+        'compatibility version to the current server version.')
+    parser.add_argument(
+        '--no-defaults', action='store_false', dest='use-defaults',
+        default=None, help='Do not use default settings; start with a minimal '
+        'number of parameters.')
+    parser.add_argument(
+        '--pip', action='append', help='A list of modules to pip install.  If '
+        'any are specified that include girder client plugins, also specify '
+        '--rebuild-client.  Each specified value is passed to pip install '
+        'directly, so additional options are needed, these can be added (such '
+        'as --find-links).  The actual values need to be escaped '
+        'appropriately for a bash shell.')
+    parser.add_argument(
+        '--rebuild-client', dest='rebuild-client', action='store_true',
+        default=False, help='Rebuild the girder client.')
+    parser.add_argument(
+        '--slicer-cli-image', dest='slicer-cli-image', action='append',
+        help='Install slicer_cli images.')
+
+    parser.add_argument(
+        '--rabbitmq-user', default='guest', dest='worker-rabbitmq-user',
+        help='Worker: RabbitMQ user name.')
+    parser.add_argument(
+        '--rabbitmq-pass', default='guest', dest='worker-rabbitmq-pass',
+        help='Worker: RabbitMQ password.')
+    parser.add_argument(
+        '--rabbitmq-host', dest='worker-rabbitmq-host',
+        help='Worker: RabbitMQ host.')
+    parser.add_argument(
+        '--config', dest='worker-config',
+        default='/opt/girder_worker/girder_worker/worker.local.cfg',
+        help='Worker: Path to the worker config file.')
+    parser.add_argument(
+        '--worker', action=YamlAction,
+        help='A yaml dictionary of worker settings.')
+    parser.add_argument(
+        '--worker-main', dest='portion', action='store_const',
+        const='worker-main',
+        help='Provision a worker, not the main process.')
+    parser.add_argument(
+        '--worker-pre', dest='portion', action='store_const',
+        const='worker-pre',
+        help='Pre-provision a worker, not the main process.')
+    parser.add_argument(
+        '--pre', dest='portion', action='store_const', const='pre',
+        help='Only do preprovisioning (install optional python modules and '
+        'optionally build the girder client).')
+    parser.add_argument(
+        '--main', dest='portion', action='store_const', const='main',
+        help='Only do main provisioning.')
+    parser.add_argument(
+        '--verbose', '-v', action='count', default=0, help='Increase verbosity')
+    parser.add_argument(
+        '--dry-run', '-n', dest='dry-run', action='store_true',
+        help='Report merged options but do not actually apply them')
+    opts = parser.parse_args(args=sys.argv[1:])
+    logger.addHandler(logging.StreamHandler(sys.stderr))
+    logger.setLevel(max(1, logging.WARNING - 10 * opts.verbose))
+    try:
+        logger.info('Provision file date: %s; size: %d',
+                    time.ctime(os.path.getmtime(__file__)),
+                    os.path.getsize(__file__))
+    except Exception:
+        pass
+    logger.debug('Parsed arguments: %r', opts)
+    if getattr(opts, 'use-defaults', None) is not False:
+        opts = merge_default_opts(opts)
+    opts = merge_yaml_opts(opts, parser)
+    opts = merge_environ_opts(opts)
+    logger.debug('Merged arguments: %r', opts)
+    if getattr(opts, 'dry-run'):
+        print(yaml.dump({k: v for k, v in vars(opts).items() if v is not None}))
+        sys.exit(0)
+    # Worker provisioning
+    if getattr(opts, 'portion', None) == 'worker-pre':
+        preprovision_worker(opts)
+        sys.exit(0)
+    if getattr(opts, 'portion', None) == 'worker-main':
+        provision_worker(opts)
+        sys.exit(0)
+    if getattr(opts, 'portion', None) in {'pre', None}:
+        # Run provisioning that has to happen before configuring the server.
+        preprovision(opts)
+        if getattr(opts, 'portion', None) == 'pre':
+            sys.exit(0)
+    if getattr(opts, 'portion', None) in {'main', None}:
+        # This loads plugins, allowing setting validation.  We want the import
+        # to be after the preprovision step.
+        from girder.utility.server import configureServer
+
+        configureServer()
+        if getattr(opts, 'mongo-compat', None) is not False:
+            from girder.models import getDbConnection
+
+            try:
+                db = getDbConnection()
+            except Exception:
+                logger.warning('Could not connect to mongo.')
+            try:
+                db.admin.command({'setFeatureCompatibilityVersion': '.'.join(
+                    db.server_info()['version'].split('.')[:2]), 'confirm': True})
+            except Exception:
+                try:
+                    db.admin.command({'setFeatureCompatibilityVersion': '.'.join(
+                        db.server_info()['version'].split('.')[:2])})
+                except Exception:
+                    logger.warning('Could not set mongo feature compatibility version.')
+            try:
+                # Also attempt to upgrade old version 2 image sources
+                db.girder.item.update_many(
+                    {'largeImage.sourceName': 'svs'},
+                    {'$set': {'largeImage.sourceName': 'openslide'}})
+            except Exception:
+                logger.warning('Could not update old source names.')
+        provision(opts)

dsa/provision.yaml

@@ -0,0 +1,100 @@
+---
+# The provision script can take a yaml file with provision options
+# This is a dictionary of command-line arguments for the provisioning script
+force: False
+samples: False
+sample-collection: Samples
+sample-folder: Images
+# Set use-defaults to False to skip default settings
+use-defaults: True
+# Set mongo_compat to False to not automatically set the mongo feature
+# compatibility version to the current server version.
+mongo-compat: True
+# A list of additional pip modules to install; if any are girder plugins with
+# client-side code, also specify rebuild-client.
+# pip:
+#   - girder-oauth
+#   - girder-ldap
+# rebuild-client may be False, True (for production mode), or "development"
+rebuild-client: False
+# Run additional shell commands before start
+# shell:
+#   - ls
+# Default admin user if there are no admin users
+admin:
+  login: admin
+  password: password
+  firstName: Admin
+  lastName: Admin
+  email: [email protected]
+  public: True
+# Default assetstore if there are no assetstores
+assetstore:
+  method: createFilesystemAssetstore
+  name: Assetstore
+  root: /assetstore
+# Any resources to ensure exist.  A model must be specified.  This creates the
+# resource if there is no match for all specified values.  A value of
+# "resource:<path>" is converted to the resource document with that resource
+# path.  "resource:admin" uses the default admin, "resourceid:<path>" is the
+# string id for the resource path, and "resourceid:admin" is the string if for
+# default admin.
+#   You can add metadata to a resource.  The default key is meta.  If
+# metadata_update is False, metadata will not be set if any metadata
+# already exists.
+resources:
+  - model: collection
+    name: Tasks
+    creator: resource:admin
+    public: True
+  - model: folder
+    parent: resource:collection/Tasks
+    parentType: collection
+    name: "Slicer CLI Web Tasks"
+    creator: resource:admin
+    public: True
+    # metadata:
+    #   sample_key: sample_value
+    # metadata_key: meta
+    # metadata_update: True
+settings:
+  worker.broker: "amqp://guest:guest@rabbitmq"
+  worker.backend: "rpc://guest:guest@rabbitmq"
+  worker.api_url: "http://girder:8080/api/v1"
+  worker.direct_path: True
+  core.brand_name: "Digital Slide Archive"
+  histomicsui.webroot_path: "histomics"
+  histomicsui.alternate_webroot_path: "histomicstk"
+  histomicsui.delete_annotations_after_ingest: True
+  homepage.markdown: |-
+    # Digital Slide Archive
+    ---
+    ## Bioinformatics Platform
+
+    Welcome to the **Digital Slide Archive**.
+
+    Developers who want to use the Girder REST API should check out the
+    [interactive web API docs](api/v1).
+
+    The [HistomicsUI](histomics) application is enabled.
+  slicer_cli_web.task_folder: "resourceid:collection/Tasks/Slicer CLI Web Tasks"
+# List slicer-cli-images to pull and load
+slicer-cli-image:
+  - dsarchive/histomicstk:latest
+# The worker can specify parameters for provisioning
+# worker-rabbitmq-host: girder:8080
+worker-rabbitmq-user: guest
+worker-rabbitmq-pass: guest
+worker-config: /opt/girder_worker/girder_worker/worker.local.cfg
+# These have precedence over the top level values
+worker:
+  # rabbitmq-host: girder:8080
+  # rabbitmq-user: guest
+  # rabbitmq-pass: guest
+  # config: /opt/girder_worker/girder_worker/worker.local.cfg
+  # Install additional pip packages in the worker
+  # pip:
+  #   - package_one
+  # Run additional shell commands in the worker before start
+  # shell:
+  #   - ls

dsa/rabbitmq.advanced.config

@@ -0,0 +1 @@
+[ {rabbit, [ {consumer_timeout, undefined} ]} ].

dsa/start_girder.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Ensures that the main process runs as the DSA_USER and is part of both that
+# group and the docker group.  Fail if DSA_USER is not specified.
+if [[ -z "$DSA_USER" ]]; then
+  echo "Set the DSA_USER before starting (e.g, DSA_USER=\$$(id -u):\$$(id -g) <up command>"
+  exit 1
+fi
+# add a user with the DSA_USER's id; this user is named ubuntu if it doesn't
+# exist.
+adduser --uid ${DSA_USER%%:*} --disabled-password --gecos "" ubuntu 2>/dev/null
+# add a group with the DSA_USER's group id.
+addgroup --gid ${DSA_USER#*:} $(id -ng ${DSA_USER#*:}) 2>/dev/null
+# add the user to the user group.
+adduser $(id -nu ${DSA_USER%%:*}) $(getent group ${DSA_USER#*:} | cut "-d:" -f1) 2>/dev/null
+# add a group with the docker group id.
+addgroup --gid $(stat -c "%g" /var/run/docker.sock) docker 2>/dev/null
+# add the user to the docker group.
+adduser $(id -nu ${DSA_USER%%:*}) $(getent group $(stat -c "%g" /var/run/docker.sock) | cut "-d:" -f1) 2>/dev/null
+# Try to increase permissions for the docker socket; this helps this work on
+# OSX where the users don't translate
+chmod 777 /var/run/docker.sock 2>/dev/null || true
+# Use iptables to make some services appear as if they are on localhost (as
+# well as on the docker network).  This is done to allow tox tests to run.
+sysctl -w net.ipv4.conf.eth0.route_localnet=1
+iptables -t nat -A OUTPUT -o lo -p tcp -m tcp --dport 27017 -j DNAT --to-destination `dig +short mongodb`:27017
+iptables -t nat -A OUTPUT -o lo -p tcp -m tcp --dport 11211 -j DNAT --to-destination `dig +short memcached`:11211
+iptables -t nat -A POSTROUTING -o eth0 -m addrtype --src-type LOCAL --dst-type UNICAST -j MASQUERADE
+echo 'PATH="/opt/digital_slide_archive/devops/dsa/utils:/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH"' >> /home/$(id -nu ${DSA_USER%%:*})/.bashrc
+echo ==== Pre-Provisioning ===
+PATH="/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH" \
+python /opt/digital_slide_archive/devops/dsa/provision.py -v --pre
+# Run subsequent commands as the DSA_USER.  This sets some paths based on what
+# is expected in the Docker so that the current python environment and the
+# devops/dsa/utils are available.  Then:
+# - Provision the Girder instance.  This sets values in the database, such as
+#   creating an admin user if there isn't one.  See the provision.py script for
+#   the details.
+# - If possible, set up a girder mount.  This allows file-like access of girder
+#   resources.  It requires the host to have fuse installed and the docker
+#   container to be run with enough permissions to use fuse.
+# - Start the main girder process.
+su $(id -nu ${DSA_USER%%:*}) -c "
+  PATH=\"/opt/digital_slide_archive/devops/dsa/utils:/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH\";
+  echo ==== Provisioning === &&
+  python /opt/digital_slide_archive/devops/dsa/provision.py -v --main &&
+  echo ==== Creating FUSE mount === &&
+  (girder mount /fuse || true) &&
+  echo ==== Starting Girder === &&
+  girder serve --dev
+"

dsa/start_worker.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Ensures that the main process runs as the DSA_USER and is part of both that
+# group and the docker group.  Fail if DSA_USER is not specified.
+if [[ -z "$DSA_USER" ]]; then
+  echo "Set the DSA_USER before starting (e.g, DSA_USER=\$$(id -u):\$$(id -g) <up command>"
+  exit 1
+fi
+# add a user with the DSA_USER's id; this user is named ubuntu if it doesn't
+# exist.
+adduser --uid ${DSA_USER%%:*} --disabled-password --gecos "" ubuntu 2>/dev/null
+# add a group with the DSA_USER's group id.
+addgroup --gid ${DSA_USER#*:} $(id -ng ${DSA_USER#*:}) 2>/dev/null
+# add the user to the user group.
+adduser $(id -nu ${DSA_USER%%:*}) $(getent group ${DSA_USER#*:} | cut "-d:" -f1) 2>/dev/null
+# add a group with the docker group id.
+addgroup --gid $(stat -c "%g" /var/run/docker.sock) docker 2>/dev/null
+# add the user to the docker group.
+adduser $(id -nu ${DSA_USER%%:*}) $(getent group $(stat -c "%g" /var/run/docker.sock) | cut "-d:" -f1) 2>/dev/null
+# Try to increase permissions for the docker socket; this helps this work on
+# OSX where the users don't translate
+chmod 777 /var/run/docker.sock 2>/dev/null || true
+chmod 777 ${TMPDIR:-/tmp} || true
+echo ==== Pre-Provisioning ===
+python3 /opt/digital_slide_archive/devops/dsa/provision.py --worker-pre
+echo ==== Provisioning === &&
+python3 /opt/digital_slide_archive/devops/dsa/provision.py --worker-main
+echo ==== Starting Worker === &&
+# Run subsequent commands as the DSA_USER.  This sets some paths based on what
+# is expected in the Docker so that the current python environment and the
+# devops/dsa/utils are available.  Then it runs girder_worker
+su $(id -nu ${DSA_USER%%:*}) -c "
+  PATH=\"/opt/digital_slide_archive/devops/dsa/utils:/opt/venv/bin:/.pyenv/bin:/.pyenv/shims:$PATH\";
+  DOCKER_CLIENT_TIMEOUT=86400 TMPDIR=${TMPDIR:-/tmp} GW_DIRECT_PATHS=true python -m girder_worker --concurrency=${DSA_WORKER_CONCURRENCY:-2} -Ofair --prefetch-multiplier=1
+"

dsa/utils/.vimrc

@@ -0,0 +1,39 @@
+" see :options
+" expandtabs
+set et
+set tabstop=4
+" shiftwidth
+set sw=4
+set nocindent
+" autoindent
+set ai
+" tell indenting programs that we already indented the buffer
+let b:did_indent = 1
+" don't do an incremental search (don't search before we finish typing)
+set nois
+" don't ignore case by default
+set noic
+" don't break at 80 characters
+set wrap
+" don't add linebreaks at 80 characters
+set nolbr
+" highlight all search matches
+set hls
+" default to utf-8
+set enc=utf-8
+" show the cursor position
+set ruler
+" allow backspace to go to the previous line
+set bs=2
+" keep this much history
+set history=50
+" don't try to maintain vi compatibility
+set nocompatible
+
+" syntax highlighting is on
+syntax on
+" save information for 100 files, with up to 50 lines for each register
+set viminfo='100,\"50
+if v:lang =~ "utf8$" || v:lang =~ "UTF-8$"
+    set fileencodings=utf-8,latin1
+endif

dsa/utils/cli_test.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python
+
+import argparse
+import getpass
+import random
+import sys
+import tempfile
+import time
+
+import girder_client
+
+
+def get_girder_client(opts):
+    """
+    Log in to Girder and return a reference to the client.
+
+    :param opts: options that include the username, password, and girder api
+        url.
+    :returns: the girder client.
+    """
+    token = opts.get('token')
+    username = opts.get('username')
+    password = opts.get('password')
+    if not username and not token:
+        username = input('Admin login: ')
+    if not password and not token:
+        password = getpass.getpass('Password for %s: ' % (
+            username if username else 'default admin user'))
+    client = girder_client.GirderClient(apiUrl=opts['apiurl'])
+    if token:
+        client.setToken(token)
+    else:
+        client.authenticate(username, password)
+    return client
+
+
+def get_test_data(client, opts):  # noqa
+    """
+    Make sure we have a test collection with a folder with test data.
+
+    :param client: girder client.
+    :param opts: command line options.
+    """
+    collName = 'HistomicsTK Tests'
+    try:
+        collection = client.resourceLookup('/collection/' + collName)
+    except Exception:
+        collection = None
+    if not collection:
+        collection = client.createCollection(collName, public=True)
+    folderName = 'Images'
+    try:
+        folder = client.resourceLookup('/collection/%s/%s' % (collName, folderName))
+    except Exception:
+        folder = None
+    if not folder:
+        folder = client.createFolder(collection['_id'], folderName, parentType='collection')
+    remote = girder_client.GirderClient(apiUrl='https://data.kitware.com/api/v1')
+    remoteFolder = remote.resourceLookup('/collection/HistomicsTK/Deployment test images')
+    for item in remote.listItem(remoteFolder['_id']):
+        localPath = '/collection/%s/%s/%s' % (collName, folderName, item['name'])
+        try:
+            localItem = client.resourceLookup(localPath)
+        except Exception:
+            localItem = None
+        if localItem:
+            if opts.get('test') == 'local':
+                continue
+            client.delete('item/%s' % localItem['_id'])
+        localItem = client.createItem(folder['_id'], item['name'])
+        for remoteFile in remote.listFile(item['_id']):
+            with tempfile.NamedTemporaryFile() as tf:
+                fileName = tf.name
+                tf.close()
+                sys.stdout.write('Downloading %s' % remoteFile['name'])
+                sys.stdout.flush()
+                remote.downloadFile(remoteFile['_id'], fileName)
+                sys.stdout.write(' .')
+                sys.stdout.flush()
+                client.uploadFileToItem(
+                    localItem['_id'], fileName, filename=remoteFile['name'],
+                    mimeType=remoteFile['mimeType'])
+                sys.stdout.write('.\n')
+                sys.stdout.flush()
+    for item in list(client.listItem(folder['_id'])):
+        if '.anot' in item['name']:
+            sys.stdout.write('Deleting %s\n' % item['name'])
+            sys.stdout.flush()
+            client.delete('item/%s' % item['_id'])
+            continue
+        if 'largeImage' not in item:
+            sys.stdout.write('Making large_item %s ' % item['name'])
+            sys.stdout.flush()
+            job = client.post('item/%s/tiles' % item['_id'])
+            if job is not None:
+                job, peak_memory = wait_for_job(client, job)
+            else:
+                print('done')
+    return folder
+
+
+def install_cli(client, imageName):
+    """
+    Make sure the specified CLI is installed.
+
+    :param client: girder client.
+    :param imageName: name of the CLI docker image
+    """
+    client.put('slicer_cli_web/docker_image', data={'name': '["%s"]' % imageName})
+    job = client.get('job/all', parameters={
+        'sort': 'created', 'sortdir': -1,
+        'types': '["slicer_cli_web_job"]',
+        'limit': 1})[0]
+    sys.stdout.write('Adding %s ' % imageName)
+    wait_for_job(client, job)
+
+
+def get_memory_use(client):
+    """
+    Get the memory use as reported by the system.
+
+    :return: the system/check virtualMemory['used'] information.
+    """
+    info = client.get('system/check?mode=quick')
+    return info['virtualMemory']['used']
+
+
+def test_cli(client, folder, opts):
+    """
+    Run the CLI on an image and make sure we get an annotation out of it.
+
+    :param client: girder client.
+    :param folder: the parent folder of the test images.
+    :param opts: command line options.
+    """
+    testItem = None
+    if not opts.get('testid'):
+        for item in client.listItem(folder['_id']):
+            if item['name'].startswith('TCGA-02'):
+                testItem = item
+                break
+    else:
+        testItem = {'_id': opts.get('testid')}
+    localFile = next(client.listFile(testItem['_id']))
+    path = 'slicer_cli_web/%s/NucleiDetection/run' % (
+        opts['cli'].replace('/', '_').replace(':', '_'), )
+    sys.stdout.write('Running %s ' % opts['cli'])
+    sys.stdout.flush()
+    anList = client.get('annotation', parameters={
+        'itemId': testItem['_id'], 'sort': '_id', 'sortdir': -1, 'limit': 1})
+    lastOldAnnotId = None
+    if len(anList):
+        lastOldAnnotId = anList[0]['_id']
+    memory_use = get_memory_use(client)
+    starttime = time.time()
+    region = '[15000,15000,1000,1000]'
+    if opts.get('randomregion'):
+        metadata = client.get('item/%s/tiles' % testItem['_id'])
+        w = metadata['sizeX']
+        h = metadata['sizeY']
+        rw = random.randint(500, 5000)
+        rh = random.randint(500, 5000)
+        region = '[%d,%d,%d,%d]' % (random.randint(0, w - rw), random.randint(0, h - rh), rw, rh)
+    if opts.get('noregion'):
+        region = '[-1,-1,-1,-1]'
+    data = {
+        'inputImageFile': localFile['_id'],
+        'outputNucleiAnnotationFile_folder': folder['_id'],
+        'outputNucleiAnnotationFile': 'cli_test.anot',
+        'analysis_roi': region,
+        'foreground_threshold': '60',
+        'min_fgnd_frac': '0.05',
+
+        'analysis_tile_size': '4096',
+        'nuclei_annotation_format': 'bbox',
+        'max_radius': '30',
+        'min_radius': '20',
+    }
+    if opts.get('testarg') and len(opts.get('testarg')):
+        testarg = {val.split('=', 1)[0]: val.split('=', 1)[1] for val in opts['testarg']}
+        data.update(testarg)
+    if opts.get('verbose', 0) >= 1:
+        sys.stdout.write('%r\n' % data)
+    job = client.post(path, data=data)
+    job, peak_memory = wait_for_job(client, job)
+    runtime = time.time() - starttime
+    # Wait for the annotation to be processed after the job finishes.
+    maxWait = time.time() + 60
+    annot = None
+    while not annot and time.time() < maxWait:
+        anList = client.get('annotation', parameters={
+            'itemId': testItem['_id'], 'sort': '_id', 'sortdir': -1, 'limit': 1})
+        if len(anList) and anList[0]['_id'] != lastOldAnnotId:
+            annot = client.get('annotation/%s' % anList[0]['_id'])
+            break
+        time.sleep(1)
+    sys.stdout.write('Total time: %5.3f, Max memory delta: %d bytes, Elements: %d\n' % (
+        runtime, peak_memory - memory_use, len(annot['annotation']['elements'])))
+    sys.stdout.flush()
+    if len(annot['annotation']['elements']) < 100:
+        raise Exception('Got less than 100 annotation elements (%d) from annotation %s' % (
+            len(annot['annotation']['elements']), anList[0]['_id']))
+
+
+def test_tiles(client, folder, opts):
+    """
+    Make sure we have a test collection with a folder with test data.
+
+    :param client: girder client.
+    :param folder: the parent folder of the test images.
+    :param opts: command line options.
+    """
+    for item in client.listItem(folder['_id']):
+        if 'largeImage' not in item:
+            raise Exception('No large image in item')
+        result = client.get('item/%s/tiles/region' % item['_id'], parameters={
+            'left': 100, 'top': 150, 'right': 400, 'bottom': 450,
+            'encoding': 'PNG',
+        }, jsonResp=False)
+        region = result.content
+        if region[1:4] != b'PNG' or len(region) < 6000:
+            raise Exception('Region did not give expected results')
+
+
+def wait_for_job(client, job):
+    """
+    Wait for a job to complete.
+
+    :param client: the girder client.
+    :param job: a girder job.
+    :return: the updated girder job.
+    """
+    peak_memory_use = get_memory_use(client)
+    lastdot = 0
+    jobId = job['_id']
+    while job['status'] not in (3, 4, 5):
+        if time.time() - lastdot >= 3:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            lastdot = time.time()
+        time.sleep(0.25)
+        peak_memory_use = max(peak_memory_use, get_memory_use(client))
+        job = client.get('job/%s' % jobId)
+    if job['status'] == 3:
+        print(' ready')
+    else:
+        print(' failed')
+    return job, peak_memory_use
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Download test data for HistomicsTK, and test that basic functions work.')
+    parser.add_argument(
+        'cli',
+        help='A cli docker image name.  This is pulled and used in tests.')
+    parser.add_argument(
+        '--apiurl', '--api', '--url', '-a',
+        default='http://127.0.0.1:8080/api/v1', help='The Girder api url.')
+    parser.add_argument(
+        '--password', '--pass', '--passwd', '--pw',
+        help='The Girder admin password.  If not specified, a prompt is given.')
+    parser.add_argument(
+        '--username', '--user',
+        help='The Girder admin username.  If not specified, a prompt is given.')
+    parser.add_argument(
+        '--token',
+        help='A Girder admin authentication token.  If specified, username '
+        'and password are ignored')
+    parser.add_argument(
+        '--no-cli', '--nocli', action='store_true', dest='nocli',
+        help='Do not pull and upload the cli; assume it is already present.')
+    parser.add_argument(
+        '--no-region', '--noregion', '--whole', action='store_true',
+        dest='noregion',
+        help='Run the cli against the whole image (this is slow).')
+    parser.add_argument(
+        '--random-region', '--randomregion', '--random', action='store_true',
+        dest='randomregion',
+        help='Run the cli against a random region on the image (this may be slow).')
+    parser.add_argument(
+        '--test', action='store_true', default=False,
+        help='Download test data and check that basic functions work.')
+    parser.add_argument(
+        '--test-local', '--local-test', '--local', action='store_const',
+        dest='test', const='local',
+        help='Use local test data and check that basic functions work.  If '
+        'local data is not present, it is downloaded.')
+    parser.add_argument(
+        '--no-test', action='store_false', dest='test',
+        help='Do not download test data and do not run checks.')
+    parser.add_argument(
+        '--test-id', dest='testid', help='The ID of the item to test.')
+    parser.add_argument(
+        '--test-arg', '--arg', '--testarg', dest='testarg', action='append',
+        help='Test arguments.  These should be of the form <key>=<value>.')
+    parser.add_argument(
+        '--only-data', '--data', action='store_const', dest='test',
+        const='data',
+        help='Download test data, but do not run CLI.')
+    parser.add_argument('--verbose', '-v', action='count', default=0)
+
+    args = parser.parse_args()
+    if args.verbose >= 2:
+        print('Parsed arguments: %r' % args)
+    client = get_girder_client(vars(args))
+    if not args.nocli:
+        install_cli(client, args.cli)
+    if args.test:
+        folder = get_test_data(client, vars(args))
+        test_tiles(client, folder, vars(args))
+        if args.test != 'data':
+            test_cli(client, folder, vars(args))

dsa/utils/rebuild_and_restart_girder.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+OLDSTART=$(curl --silent 'http://127.0.0.1:8080/api/v1/system/version')
+girder build --dev
+touch /etc/girder.cfg
+echo "Girder has been rebuilt and will now restart"
+while true; do NEWSTART=$(curl --silent 'http://127.0.0.1:8080/api/v1/system/version' || true); if [ "${OLDSTART}" != "${NEWSTART}" ]; then echo ${NEWSTART} | grep -q 'release' && break || true; fi; sleep 1; echo -n "."; done
+echo ""
+echo "Girder has restarted"

dsa/utils/restart_girder.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+OLDSTART=$(curl --silent 'http://127.0.0.1:8080/api/v1/system/version')
+touch /etc/girder.cfg
+echo "Girder will now restart"
+while true; do NEWSTART=$(curl --silent 'http://127.0.0.1:8080/api/v1/system/version' || true); if [ "${OLDSTART}" != "${NEWSTART}" ]; then echo ${NEWSTART} | grep -q 'release' && break || true; fi; sleep 1; echo -n "."; done
+echo ""
+echo "Girder has restarted"