Upload folder using huggingface_hub

.dockerignore

@@ -0,0 +1,17 @@
+.git
+.npm
+.dockerignore
+.pytest_cache
+.cache
+.local
+.github
+.nv
+.benchmarks
+.bash_history
+.gitignore
+h2ogpt.egg-info
+venv
+build
+dist
+prebuilt_deps
+Dockerfile

.env

@@ -0,0 +1,20 @@
+
+# H2OGPT
+
+H2OGPT_PORT=7860
+H2OGPT_BASE_MODEL=h2oai/h2ogpt-4096-llama2-7b-chat
+H2OGPT_ARGS="/workspace/generate.py --base_model=${H2OGPT_BASE_MODEL} --use_safetensors=True --prompt_type=llama2 --save_dir=/workspace/save/ --use_gpu_id=False --score_model=None --max_max_new_tokens=2048 --max_new_tokens=1024"
+
+# VLLM
+
+VLLM_TOKENIZER=hf-internal-testing/llama-tokenizer
+H2OGPT_VLLM_ARGS="--model=${H2OGPT_BASE_MODEL} --tokenizer=${VLLM_TOKENIZER} --tensor-parallel-size=2 --seed=1234 --trust-remote-code --download-dir=/workspace/.cache/huggingface/hub"
+
+# CPU models
+
+MODEL_PATH_LLAMA=https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q6_K.gguf
+H2OGPT_CPU_ARGS="/workspace/generate.py --base_model=llama --model_path_llama=${MODEL_PATH_LLAMA} --max_seq_len=4096"
+
+# API KEY
+
+OPENAI_API_KEY = "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"

.gitattributes

@@ -1,35 +1,28 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+data/count_1w.txt.zip filter=lfs diff=lfs merge=lfs -text
+data/demo.png filter=lfs diff=lfs merge=lfs -text
+data/NGSL_1.2_stats.csv.zip filter=lfs diff=lfs merge=lfs -text
+docs/aws_sagemaker_endpoint_setup.pdf filter=lfs diff=lfs merge=lfs -text
+docs/ffmpeg.7z filter=lfs diff=lfs merge=lfs -text
+docs/jpeg-6b-4-bin.zip filter=lfs diff=lfs merge=lfs -text
+docs/jpeg-6b-4-dep.zip filter=lfs diff=lfs merge=lfs -text
+docs/libpng-1.2.37-bin.zip filter=lfs diff=lfs merge=lfs -text
+docs/libpng-1.2.37-dep.zip filter=lfs diff=lfs merge=lfs -text
+ffmpeg.7z filter=lfs diff=lfs merge=lfs -text
+jpeg-6b-4-bin.zip filter=lfs diff=lfs merge=lfs -text
+jpeg-6b-4-dep.zip filter=lfs diff=lfs merge=lfs -text
+libpng-1.2.37-bin.zip filter=lfs diff=lfs merge=lfs -text
+libpng-1.2.37-dep.zip filter=lfs diff=lfs merge=lfs -text
+llamacpp_path/llama-2-7b-chat.Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_awb_arctic-wav-arctic_a0002.npy filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_bdl_arctic-wav-arctic_a0009.npy filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_clb_arctic-wav-arctic_a0144.npy filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_ksp_arctic-wav-arctic_b0087.npy filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_rms_arctic-wav-arctic_b0353.npy filter=lfs diff=lfs merge=lfs -text
+spkemb/cmu_us_slt_arctic-wav-arctic_a0508.npy filter=lfs diff=lfs merge=lfs -text
+tests/CityofTshwaneWater.pdf filter=lfs diff=lfs merge=lfs -text
+tests/driverslicense.jpeg.zip filter=lfs diff=lfs merge=lfs -text
+tests/fastfood.jpg filter=lfs diff=lfs merge=lfs -text
+tests/ocr2.png filter=lfs diff=lfs merge=lfs -text
+tests/porsche.mp3.zip filter=lfs diff=lfs merge=lfs -text
+tests/test_speech.wav filter=lfs diff=lfs merge=lfs -text
+tests/videotest.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,44 @@
+out/
+7B/
+13B/
+__pycache__/
+checkpoint**
+minimal-llama**
+upload.py
+lora-**
+*ckpt
+wandb
+evaluate.py
+test_data.json
+todo.txt
+.neptune/
+*.bin
+db_dir_UserData
+temp_path_do_doc1
+offline_folder
+flagged_data_points
+.pytest_cache
+user_path
+user_path_test
+build
+h2ogpt.egg-info
+dist
+.idea
+.cache
+.local
+.bash_history
+.benchmarks
+Dockerfile-runner.dockerfile
+build_info.txt
+prebuilt_deps
+Dockerfile_deps
+
+# IDEs
+.idea/
+
+# virtual envs
+venv
+
+# Mac one click installer
+Tesseract-OCR/
+poppler/

Dockerfile

@@ -0,0 +1,46 @@
+# devel needed for bitsandbytes requirement of libcudart.so, otherwise runtime sufficient
+FROM nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV PATH="/h2ogpt_conda/envs/h2ogpt/bin:${PATH}"
+ARG PATH="/h2ogpt_conda/envs/h2ogpt/bin:${PATH}"
+
+ENV HOME=/workspace
+ENV CUDA_HOME=/usr/local/cuda-12.1
+ENV VLLM_CACHE=/workspace/.vllm_cache
+ENV TIKTOKEN_CACHE_DIR=/workspace/tiktoken_cache
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
+
+WORKDIR /workspace
+
+COPY . /workspace/
+
+COPY build_info.txt /workspace/
+
+RUN cd /workspace && ./docker_build_script_ubuntu.sh
+
+RUN chmod -R a+rwx /workspace
+
+ARG user=h2ogpt
+ARG group=h2ogpt
+ARG uid=1000
+ARG gid=1000
+
+RUN groupadd -g ${gid} ${group} && useradd -u ${uid} -g ${group} -s /bin/bash ${user}
+# already exists in base image
+# RUN groupadd -g ${gid} docker && useradd -u ${uid} -g ${group} -m ${user}
+
+# Add the user to the docker group
+RUN usermod -aG docker ${user}
+
+# Switch to the new user
+USER ${user}
+
+EXPOSE 8888
+EXPOSE 7860
+EXPOSE 5000
+EXPOSE 5002
+EXPOSE 5004
+
+ENTRYPOINT ["python3.10"]

LICENSE

@@ -0,0 +1,201 @@
+                                Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,94 @@
+all: clean dist
+
+PACKAGE_VERSION              := `cat version.txt | tr -d '\n'`
+BUILD_TAG                    := $(shell git describe --always --dirty)
+DOCKER_H2OGPT_RUNTIME_IMAGE  := gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(BUILD_TAG)
+DOCKER_H2OGPT_VLLM_IMAGE     := gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(BUILD_TAG)
+PYTHON_BINARY                ?= `which python`
+DEFAULT_MARKERS              ?= "not need_tokens and not need_gpu"
+
+# h2ogpt base and vllm images built elsewhere and referenced here:
+DOCKER_BASE_OS_IMAGE     := gcr.io/vorvan/h2oai/h2ogpt-oss-wolfi-base:9
+DOCKER_VLLM_IMAGE        := gcr.io/vorvan/h2oai/h2ogpte-vllm:0.6.3.post1-38ed4ff2
+
+
+.PHONY: venv dist test publish docker_build docker_push build_info.txt
+
+clean:
+	rm -rf dist build h2ogpt.egg-info
+
+venv:
+	$(PYTHON_BINARY) -m virtualenv -p $(PYTHON_BINARY) venv
+
+install:
+	$(PYTHON_BINARY) -m pip install dist/h2ogpt-$(PACKAGE_VERSION)-py3-none-any.whl
+
+install-%:
+	$(PYTHON_BINARY) -m pip install dist/h2ogpt-$(PACKAGE_VERSION)-py3-none-any.whl[$*]
+
+dist:
+	$(PYTHON_BINARY) setup.py bdist_wheel
+
+test:
+	$(PYTHON_BINARY) -m pip install requirements-parser
+	$(PYTHON_BINARY) -m pytest tests --disable-warnings --junit-xml=test_report.xml -m "$(DEFAULT_MARKERS)"
+
+test_imports:
+	$(PYTHON_BINARY) -m pytest tests/test_imports.py --disable-warnings --junit-xml=test_report.xml -m "$(DEFAULT_MARKERS)"
+
+publish:
+	echo "Publishing not implemented yet."
+
+build_info.txt:
+	@rm -rf build_info.txt
+	@echo "commit=\"$(shell git rev-parse HEAD)\"" >> $@
+	@echo "branch=\"`git rev-parse HEAD | git branch -a --contains | grep -v detached | sed -e 's~remotes/origin/~~g' -e 's~^ *~~' | sort | uniq | tr '*\n' ' '`\"" >> $@
+	@echo "describe=\"`git describe --always --dirty`\"" >> $@
+	@echo "build_os=\"`uname -a`\"" >> $@
+	@echo "build_machine=\"`hostname`\"" >> $@
+	@echo "build_date=\"$(shell date "+%Y%m%d")\"" >> $@
+	@echo "build_user=\"`id -u -n`\"" >> $@
+	@echo "base_version=\"$(PACKAGE_VERSION)\"" >> $@
+
+
+docker_build: build_info.txt
+ifeq ($(shell curl --connect-timeout 4 --write-out %{http_code} -sS --output /dev/null -X GET https://gcr.io/v2/vorvan/h2oai/h2oai-h2ogpt-runtime/manifests/$(BUILD_TAG)),200)
+	@echo "Image already pushed to GCR: $(DOCKER_H2OGPT_RUNTIME_IMAGE)"
+	docker pull $(DOCKER_H2OGPT_RUNTIME_IMAGE)
+else
+	docker pull $(DOCKER_BASE_OS_IMAGE)
+	DOCKER_BUILDKIT=1 docker build -t $(DOCKER_H2OGPT_RUNTIME_IMAGE) -t h2ogpt:current -f Dockerfile .
+endif
+ifeq ($(shell curl --connect-timeout 4 --write-out %{http_code} -sS --output /dev/null -X GET https://gcr.io/v2/vorvan/h2oai/h2oai-h2ogpt-vllm/manifests/$(BUILD_TAG)),200)
+	@echo "VLLM Image already pushed to GCR: $(DOCKER_H2OGPT_VLLM_IMAGE)"
+	docker pull $(DOCKER_H2OGPT_VLLM_IMAGE)
+else
+	docker pull $(DOCKER_VLLM_IMAGE)
+	docker tag $(DOCKER_VLLM_IMAGE) $(DOCKER_H2OGPT_VLLM_IMAGE)
+endif
+
+docker_push:
+	docker tag $(DOCKER_H2OGPT_RUNTIME_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(PACKAGE_VERSION)
+	docker tag $(DOCKER_H2OGPT_VLLM_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(PACKAGE_VERSION)
+
+	docker tag $(DOCKER_H2OGPT_RUNTIME_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:latest
+	docker tag $(DOCKER_H2OGPT_VLLM_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:latest
+
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(BUILD_TAG)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(PACKAGE_VERSION)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:latest
+
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(BUILD_TAG)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(PACKAGE_VERSION)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:latest
+
+ifdef BUILD_ID
+	docker tag $(DOCKER_H2OGPT_RUNTIME_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(PACKAGE_VERSION)-$(BUILD_ID)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-runtime:$(PACKAGE_VERSION)-$(BUILD_ID)
+
+	docker tag $(DOCKER_H2OGPT_VLLM_IMAGE) gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(PACKAGE_VERSION)-$(BUILD_ID)
+	docker push gcr.io/vorvan/h2oai/h2oai-h2ogpt-vllm:$(PACKAGE_VERSION)-$(BUILD_ID)
+endif
+
+print-%:
+	@echo $($*)

README.md

@@ -1,12 +1,215 @@
 ---
-title: Aiben
-emoji: 📊
-colorFrom: blue
-colorTo: pink
+title: aiben
+app_file: generate.py
 sdk: gradio
-sdk_version: 5.9.1
-app_file: app.py
-pinned: false
+sdk_version: 4.44.0
 ---
+# h2oGPT
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Query and summarize your documents or just chat with local private GPT LLMs using h2oGPT, an Apache V2 open-source project.
+
+## Live Demo
+
+[![img-small.png](docs/img-small.png) Gradio Demo](https://gpt.h2o.ai/)
+
+[![img-small.png](docs/img-small.png) OpenWebUI Demo](https://gpt-docs.h2o.ai/)
+
+## Video Demo
+
+https://github.com/h2oai/h2ogpt/assets/2249614/2f805035-2c85-42fb-807f-fd0bca79abc6
+
+[![img-small.png](docs/img-small.png) YouTube 4K Video](https://www.youtube.com/watch?v=_iktbj4obAI)
+
+## Features
+
+- **Private** offline database of any documents [(PDFs, Excel, Word, Images, Video Frames, YouTube, Audio, Code, Text, MarkDown, etc.)](docs/README_LangChain.md#supported-datatypes)
+  - **Persistent** database (Chroma, Weaviate, or in-memory FAISS) using accurate embeddings (instructor-large, all-MiniLM-L6-v2, etc.)
+  - **Efficient** use of context using instruct-tuned LLMs (no need for LangChain's few-shot approach)
+  - **Parallel** summarization and extraction, reaching an output of 80 tokens per second with the 13B LLaMa2 model
+  - **HYDE** (Hypothetical Document Embeddings) for enhanced retrieval based upon LLM responses
+  - **Semantic Chunking** for better document splitting (requires GPU)
+- **Variety** of models supported (LLaMa2, Mistral, Falcon, Vicuna, WizardLM.  With AutoGPTQ, 4-bit/8-bit, LORA, etc.)
+  - **GPU** support from HF and LLaMa.cpp GGML models, and **CPU** support using HF, LLaMa.cpp, and GPT4ALL models
+  - **Attention Sinks** for [arbitrarily long](https://github.com/tomaarsen/attention_sinks) generation (LLaMa-2, Mistral, MPT, Pythia, Falcon, etc.)
+- **Gradio UI** or CLI with streaming of all models
+  - **Upload** and **View** documents through the UI (control multiple collaborative or personal collections)
+  - **Vision Models** LLaVa, Claude-3, Gemini-Pro-Vision, GPT-4-Vision
+  - **Image Generation** Stable Diffusion (sdxl-turbo, sdxl, SD3), PlaygroundAI (playv2), and Flux
+  - **Voice STT** using Whisper with streaming audio conversion
+  - **Voice TTS** using MIT-Licensed Microsoft Speech T5 with multiple voices and Streaming audio conversion
+  - **Voice TTS** using MPL2-Licensed TTS including Voice Cloning and Streaming audio conversion
+  - **AI Assistant Voice Control Mode** for hands-free control of h2oGPT chat
+  - **Bake-off** UI mode against many models at the same time
+  - **Easy Download** of model artifacts and control over models like LLaMa.cpp through the UI
+  - **Authentication** in the UI by user/password via Native or Google OAuth
+  - **State Preservation** in the UI by user/password
+- **Open Web UI** with h2oGPT as backend via OpenAI Proxy
+  - See [Start-up Docs](docs/FAQ.md#open-web-ui).
+  - Chat completion with streaming
+  - Document Q/A using h2oGPT ingestion with advanced OCR from DocTR
+  - Vision models
+  - Audio Transcription (STT)
+  - Audio Generation (TTS)
+  - Image generation
+  - Authentication
+  - State preservation
+- **Linux, Docker, macOS, and Windows** support
+- **Inference Servers** [support](docs/README_InferenceServers.md) for oLLaMa, HF TGI server, vLLM, Gradio, ExLLaMa, Replicate, Together.ai, OpenAI, Azure OpenAI, Anthropic, MistralAI, Google, and Groq
+- **OpenAI compliant**
+  - Server Proxy [API](docs/README_CLIENT.md) (h2oGPT acts as drop-in-replacement to OpenAI server)
+  - Chat and Text Completions (streaming and non-streaming)
+  - Audio Transcription (STT)
+  - Audio Generation (TTS)
+  - Image Generation
+  - Embedding
+  - Function tool calling w/auto tool selection
+  - AutoGen Code Execution Agent
+- **JSON Mode**
+  - Strict schema control for vLLM via its use of outlines
+  - Strict schema control for OpenAI, Anthropic, Google Gemini, MistralAI models
+  - JSON mode for some older OpenAI or Gemini models with schema control if model is smart enough (e.g. gemini 1.5 flash)
+  - Any model via code block extraction
+- **Web-Search** integration with Chat and Document Q/A
+- **Agents** for Search, Document Q/A, Python Code, CSV frames
+  - High quality Agents via OpenAI proxy server on separate port
+  - Code-first agent that generates plots, researches, evaluates images via vision model, etc. (client code openai_server/openai_client.py).
+  - No UI for this, just API
+- **Evaluate** performance using reward models
+- **Quality** maintained with over 1000 unit and integration tests taking over 24 GPU-hours
+
+## Get Started
+
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-docker?style=flat-square)](LICENSE)
+[![Linux](https://img.shields.io/badge/Linux-FCC624?style=for-the-badge&logo=linux&logoColor=black)](https://github.com/h2oai/h2ogpt/blob/main/docs/README_LINUX.md)
+[![macOS](https://img.shields.io/badge/mac%20os-000000?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://github.com/h2oai/h2ogpt/blob/main/docs/README_MACOS.md)
+[![Windows](https://img.shields.io/badge/Windows-0078D6?style=for-the-badge&logo=windows&logoColor=white)](https://github.com/h2oai/h2ogpt/blob/main/docs/README_WINDOWS.md)
+[![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?style=for-the-badge&logo=docker&logoColor=white)](https://github.com/h2oai/h2ogpt/blob/main/docs/README_DOCKER.md)
+
+### Install h2oGPT
+
+Docker is recommended for Linux, Windows, and MAC for full capabilities.  Linux Script also has full capability, while Windows and MAC scripts have less capabilities than using Docker.
+
+* [Docker Build and Run Docs (Linux, Windows, MAC)](docs/README_DOCKER.md)
+* [Linux Install and Run Docs](docs/README_LINUX.md)
+* [Windows 10/11 Installation Script](docs/README_WINDOWS.md)
+* [MAC Install and Run Docs](docs/README_MACOS.md)
+* [Quick Start on any Platform](docs/README_quickstart.md)
+
+---
+
+### Collab Demos
+- [![](https://colab.research.google.com/assets/colab-badge.svg) h2oGPT CPU](https://colab.research.google.com/drive/13RiBdAFZ6xqDwDKfW6BG_-tXfXiqPNQe?usp=sharing)
+- [![](https://colab.research.google.com/assets/colab-badge.svg) h2oGPT GPU](https://colab.research.google.com/drive/143-KFHs2iCqXTQLI2pFCDiR69z0dR8iE?usp=sharing)
+
+### Resources
+- [FAQs](docs/FAQ.md)
+- [README for LangChain](docs/README_LangChain.md)
+- [Discord](https://discord.gg/WKhYMWcVbq)
+- [Models (LLaMa-2, Falcon 40, etc.) at 🤗](https://huggingface.co/h2oai/)
+- [YouTube: 100% Offline ChatGPT Alternative?](https://www.youtube.com/watch?v=Coj72EzmX20)
+- [YouTube: Ultimate Open-Source LLM Showdown (6 Models Tested) - Surprising Results!](https://www.youtube.com/watch?v=FTm5C_vV_EY)
+- [YouTube: Blazing Fast Falcon 40b 🚀 Uncensored, Open-Source, Fully Hosted, Chat With Your Docs](https://www.youtube.com/watch?v=H8Dx-iUY49s)
+- [Technical Paper: https://arxiv.org/pdf/2306.08161.pdf](https://arxiv.org/pdf/2306.08161.pdf)
+
+### Docs Guide
+<!--  cat README.md | ./gh-md-toc  -  But Help is heavily processed -->
+* [Get Started](#get-started)
+   * [Linux (CPU or CUDA)](docs/README_LINUX.md)
+   * [macOS (CPU or M1/M2)](docs/README_MACOS.md)
+   * [Windows 10/11 (CPU or CUDA)](docs/README_WINDOWS.md)
+   * [GPU (CUDA, AutoGPTQ, exllama) Running Details](docs/README_GPU.md)
+   * [CPU Running Details](docs/README_CPU.md)
+   * [CLI chat](docs/README_CLI.md)
+   * [Gradio UI](docs/README_ui.md)
+   * [Client API (Gradio, OpenAI-Compliant)](docs/README_CLIENT.md)
+   * [Inference Servers (oLLaMa, HF TGI server, vLLM, Groq, Anthropic, Google, Mistral, Gradio, ExLLaMa, Replicate, OpenAI, Azure OpenAI)](docs/README_InferenceServers.md)
+   * [Build Python Wheel](docs/README_WHEEL.md)
+   * [Offline Installation](docs/README_offline.md)
+   * [Low Memory](docs/FAQ.md#low-memory-mode)
+   * [Docker](docs/README_DOCKER.md)
+* [LangChain Document Support](docs/README_LangChain.md)
+* [Compare to PrivateGPT et al.](docs/README_LangChain.md#what-is-h2ogpts-langchain-integration-like)
+* [Roadmap](#roadmap)
+* [Development](#development)
+* [Help](#help)
+   * [LangChain file types supported](docs/README_LangChain.md#supported-datatypes)
+   * [CLI Database control](docs/README_LangChain.md#database-creation)
+   * [FAQ](docs/FAQ.md)
+     * [Model Usage Notes](docs/FAQ.md#model-usage-notes)
+     * [Adding LLM Models (including using GGUF and Attention Sinks)](docs/FAQ.md#adding-models)
+     * [Adding Embedding Models](docs/FAQ.md#add-new-embedding-model)
+     * [Adding Prompts](docs/FAQ.md#adding-prompt-templates)
+     * [In-Context Learning](docs/FAQ.md#in-context-learning-via-prompt-engineering)
+     * [Multiple GPUs](docs/FAQ.md#multiple-gpus)
+     * [Low-Memory Usage](docs/FAQ.md#low-memory-mode)
+     * [Environment Variables](docs/FAQ.md#what-envs-can-i-pass-to-control-h2ogpt)
+     * [HTTPS access for server and client](docs/FAQ.md#https-access-for-server-and-client)
+   * [Useful Links](docs/LINKS.md)
+   * [Fine-Tuning](docs/FINETUNE.md)
+   * [Triton](docs/TRITON.md)
+   * [Commercial viability](docs/FAQ.md#commercial-viability)
+* [Acknowledgements](#acknowledgements)
+* [Why H2O.ai?](#why-h2oai)
+* [Disclaimer](#disclaimer)
+
+### Development
+
+- To create a development environment for training and generation, follow the [installation instructions](docs/INSTALL.md).
+- To fine-tune any LLM models on your data, follow the [fine-tuning instructions](docs/FINETUNE.md).
+- To run h2oGPT tests:
+    ```bash
+    pip install requirements-parser pytest-instafail pytest-random-order playsound==1.3.0
+    conda install -c conda-forge gst-python -y
+    sudo apt-get install gstreamer-1.0
+    pip install pygame
+    GPT_H2O_AI=0 CONCURRENCY_COUNT=1 pytest --instafail -s -v tests
+    # for openai server test on already-running local server
+    pytest -s -v -n 4 openai_server/test_openai_server.py::test_openai_client
+    ```
+  or tweak/run `tests/test4gpus.sh` to run tests in parallel.
+
+### Acknowledgements
+
+* Some training code was based upon March 24 version of [Alpaca-LoRA](https://github.com/tloen/alpaca-lora/).
+* Used high-quality created data by [OpenAssistant](https://open-assistant.io/).
+* Used base models by [EleutherAI](https://www.eleuther.ai/).
+* Used OIG data created by [LAION](https://laion.ai/blog/oig-dataset/).
+
+### Why H2O.ai?
+
+Our [Makers](https://h2o.ai/company/team/) at [H2O.ai](https://h2o.ai) have built several world-class Machine Learning, Deep Learning and AI platforms:
+- #1 open-source machine learning platform for the enterprise [H2O-3](https://github.com/h2oai/h2o-3)
+- The world's best AutoML (Automatic Machine Learning) with [H2O Driverless AI](https://h2o.ai/platform/ai-cloud/make/h2o-driverless-ai/)
+- No-Code Deep Learning with [H2O Hydrogen Torch](https://h2o.ai/platform/ai-cloud/make/hydrogen-torch/)
+- Document Processing with Deep Learning in [Document AI](https://h2o.ai/platform/ai-cloud/make/document-ai/)
+
+We also built platforms for deployment and monitoring, and for data wrangling and governance:
+- [H2O MLOps](https://h2o.ai/platform/ai-cloud/operate/h2o-mlops/) to deploy and monitor models at scale
+- [H2O Feature Store](https://h2o.ai/platform/ai-cloud/make/feature-store/) in collaboration with AT&T
+- Open-source Low-Code AI App Development Frameworks [Wave](https://wave.h2o.ai/) and [Nitro](https://nitro.h2o.ai/)
+- Open-source Python [datatable](https://github.com/h2oai/datatable/) (the engine for H2O Driverless AI feature engineering)
+
+Many of our customers are creating models and deploying them enterprise-wide and at scale in the [H2O AI Cloud](https://h2o.ai/platform/ai-cloud/):
+- Multi-Cloud or on Premises
+- [Managed Cloud (SaaS)](https://h2o.ai/platform/ai-cloud/managed)
+- [Hybrid Cloud](https://h2o.ai/platform/ai-cloud/hybrid)
+- [AI Appstore](https://docs.h2o.ai/h2o-ai-cloud/)
+
+We are proud to have over 25 (of the world's 280) [Kaggle Grandmasters](https://h2o.ai/company/team/kaggle-grandmasters/) call H2O home, including three Kaggle Grandmasters who have made it to world #1.
+
+### Disclaimer
+
+Please read this disclaimer carefully before using the large language model provided in this repository. Your use of the model signifies your agreement to the following terms and conditions.
+
+- Biases and Offensiveness: The large language model is trained on a diverse range of internet text data, which may contain biased, racist, offensive, or otherwise inappropriate content. By using this model, you acknowledge and accept that the generated content may sometimes exhibit biases or produce content that is offensive or inappropriate. The developers of this repository do not endorse, support, or promote any such content or viewpoints.
+- Limitations: The large language model is an AI-based tool and not a human. It may produce incorrect, nonsensical, or irrelevant responses. It is the user's responsibility to critically evaluate the generated content and use it at their discretion.
+- Use at Your Own Risk: Users of this large language model must assume full responsibility for any consequences that may arise from their use of the tool. The developers and contributors of this repository shall not be held liable for any damages, losses, or harm resulting from the use or misuse of the provided model.
+- Ethical Considerations: Users are encouraged to use the large language model responsibly and ethically. By using this model, you agree not to use it for purposes that promote hate speech, discrimination, harassment, or any form of illegal or harmful activities.
+- Reporting Issues: If you encounter any biased, offensive, or otherwise inappropriate content generated by the large language model, please report it to the repository maintainers through the provided channels. Your feedback will help improve the model and mitigate potential issues.
+- Changes to this Disclaimer: The developers of this repository reserve the right to modify or update this disclaimer at any time without prior notice. It is the user's responsibility to periodically review the disclaimer to stay informed about any changes.
+
+By using the large language model provided in this repository, you agree to accept and comply with the terms and conditions outlined in this disclaimer. If you do not agree with any part of this disclaimer, you should refrain from using the model and any content generated by it.
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=h2oai/h2ogpt&type=Timeline)](https://star-history.com/#h2oai/h2ogpt&Timeline)

aiben_inputs.csv

@@ -0,0 +1,11 @@
+questions,context_prompt,eval_prompt
+"1. Compliance with Anti-Money Laundering (AML) Regulations: Does the organization have a robust AML program in place, including a designated AML compliance officer? Ensure that there is a designated person responsible for managing AML policies and compliance.
+2. Know Your Customer (KYC) Procedures: Are all customers subject to KYC checks, including identity verification and background screening? Review the processes used to verify customer identities, including the use of government-issued identification and any third-party background checks.
+3. Suspicious Activity Reporting (SAR): Has the organization filed Suspicious Activity Reports (SARs) for any transactions that appear suspicious or fall under AML regulations? Check the organization's SAR filing history to ensure timely and accurate reports of suspicious activities, such as large, irregular transactions.
+4. Transaction Monitoring and Reporting: Does the organization have an automated system to monitor transactions for suspicious activity and compliance with transaction reporting thresholds? Confirm whether automated systems are in place for detecting suspicious activity, such as unusually large transactions or transactions from high-risk jurisdictions.
+5. Dodd-Frank Act Compliance: Consumer Protection: Does the organization comply with the consumer protection requirements of the Dodd-Frank Wall Street Reform and Consumer Protection Act? Check for compliance with regulations such as the Consumer Financial Protection Bureau (CFPB) rules on lending practices, and confirm that the institution is avoiding deceptive or unfair practices.
+6. Fair Lending Practices (Equal Credit Opportunity Act): Is the organization adhering to fair lending practices, ensuring that loans are not denied based on discriminatory factors such as race, gender, or ethnicity? Verify that all loans are processed in compliance with the Equal Credit Opportunity Act (ECOA), which ensures fair and non-discriminatory lending.
+7. Loan Underwriting and Risk Assessment Procedures: Are loan underwriting procedures consistent with the organization’s internal policies and regulatory guidelines, including thorough risk assessments? Review whether loan approvals are based on sound financial data, such as the applicant’s credit score, income, debt-to-income ratio, and the value of any collateral.
+8. Transparency in Loan Terms (Truth in Lending Act - TILA): Does the organization provide full and transparent disclosure of loan terms, including APR, interest rates, fees, and repayment schedules, in compliance with the Truth in Lending Act (TILA)? Ensure that the organization is adhering to TILA by providing clear and accurate loan terms before borrowers sign any agreements.
+9. Documentation and Record-Keeping: Is the organization maintaining proper records of loan documents, customer interactions, and compliance activities for the required retention period (typically 5 years)?  Check that all loan documentation, customer communications, and compliance checks are properly stored and readily accessible for audits and inspections.
+10. Regulatory Reporting and Audit Trail: Does the organization provide accurate and timely regulatory reports, including required financial disclosures, tax filings, and audit trails of transactions and loan decisions? Review whether the organization is submitting required regulatory reports on time, ensuring proper documentation and transparency for financial audits, regulatory reviews, and internal inspections.",Read the following document and answer the following question clearly and concisely in 100 words or less. SAMPLE DOCUMENT: {sample_plan} QUESTION: {subquestion} RELEVANT CONTEXT: {subquestion_context} ANSWER: ,"According to government policy, an acceptable document must have all of the elements described in the following questions. Read the following question-and-answer pairs about a certain proposal and determine whether or not it conforms to the government pollicy. Remember: if any single element is missing from the proposal, it automatically means that the entire proposal does NOT conform to government policy. If the plan does not conform to government policy, explain why not. {qa_pairs} Based on the question-and-answer pairs above, does the plan follow government policy?"

benchmarks/llm_gpu_benchmark.py

@@ -0,0 +1,123 @@
+
+
+# %%
+import json
+
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import plotly.io as pio
+from plotly.subplots import make_subplots
+
+# %%
+# Read the json file
+# This file processes the llm_gpu_benchmark.json file in the tmp/inputs folder
+# File is generated using the command
+# curl  -sSL https://raw.githubusercontent.com/h2oai/h2ogpt/main/benchmarks/perf.json | jq -s '.' > llm_gpu_benchmarks.json
+with open('llm_gpu_benchmarks.json') as f:
+    data = json.load(f)
+del f
+
+# %%
+# Read the json file into a dataframe
+df = pd.json_normalize(data)
+del data
+
+# %%
+# Process the dataframe
+# Drop columns that are not needed
+df.drop(columns=['task', 'ngpus', 'reps', 'date', 'git_sha', 'transformers', 'bitsandbytes', 'cuda', 'hostname',
+                 'summarize_input_len_bytes'], inplace=True)
+# Rename columns
+df.rename(columns={'n_gpus': 'gpu_count'}, inplace=True)
+# Split the gpu column into gpu and gpu_memory
+df["gpu_name"] = df.gpus.str.extract(r'[1-9] x ([\w\- ]+) .+')
+df["gpu_memory_gb"] = round(
+    pd.to_numeric(df.gpus.str.extract(r'[\w ]+ \(([\d]+) .+', expand=False), errors='coerce') / 1024)
+df["gpu_memory_gb"] = df["gpu_memory_gb"].astype('Int64')
+df.drop(columns=['gpus'], inplace=True)
+# Manage gpu_names
+df.gpu_name = df.gpu_name.str.replace('NVIDIA ', '')
+df.gpu_name = df.gpu_name.str.replace('GeForce ', '')
+df.gpu_name = df.gpu_name.str.replace('A100-SXM4-80GB', 'A100 SXM4')
+df.gpu_name = df.gpu_memory_gb.astype(str) + "-" + df.gpu_name
+# Remove CPUs
+df.drop(df[df.gpu_name.isnull()].index, inplace=True)
+
+# %%
+# Remove duplicate rows
+df.drop_duplicates(['backend', 'base_model', 'bits', 'gpu_count', 'gpu_name'], inplace=True)
+
+# %% Add baseline comparison columns
+# Looking at the CPU data for 4, 8, and 16 bit quantization values for the benchmark we are simplifying it to a single
+# value
+cpu_summary_out_throughput = 1353 / 1216  # bytes/second  (calculated from summarize_output_len_bytes / summarize_time)
+cpu_generate_out_throughput = 849 / 180  # bytes/second   (calculated from generate_output_len_bytes / generate_time)
+
+# add GPU throughput columns
+df["summary_out_throughput"] = df.summarize_output_len_bytes / df.summarize_time
+df["generate_out_throughput"] = df.generate_output_len_bytes / df.generate_time
+# add GPU throughput boost columns
+df["summary_out_throughput_normalize"] = df.summary_out_throughput / cpu_summary_out_throughput
+df["generate_out_throughput_normalize"] = df.generate_out_throughput / cpu_generate_out_throughput
+
+# %%
+# df.to_excel('tmp/scratchpad/output/llm_gpu_benchmarks.xlsx', index=False)
+
+# %%
+pio.renderers.default = "browser"
+
+# %%
+bits_bar_colors = {'4': px.colors.qualitative.D3[0],
+                   '8': px.colors.qualitative.D3[1],
+                   '16': px.colors.qualitative.D3[2]}
+
+backends = list(df.backend.unique())
+base_models = list(df.base_model.unique())
+n_gpus = list(df.gpu_count.unique())
+
+# %%
+for backend in backends:
+    # for backend in ['transformers']:
+    fig_bar = make_subplots(rows=len(n_gpus),
+                            cols=len(base_models) * 2,
+                            shared_xaxes='all',
+                            shared_yaxes='columns',
+                            start_cell="top-left",
+                            vertical_spacing=0.1,
+                            print_grid=False,
+                            row_titles=[f'{gpu_count} GPUs' for gpu_count in n_gpus],
+                            column_titles=['llama2-7b-chat Summarization', 'llama2-7b-chat Generation',
+                                           'llama2-13b-chat Summarization', 'llama2-13b-chat Generation',
+                                           'llama2-70b-chat Summarization', 'llama2-70b-chat Generation'],)
+
+    # for base_model in ['h2oai/h2ogpt-4096-llama2-7b-chat']:
+    for base_model in base_models:
+        for gpu_count in n_gpus:
+            for bits in sorted(df.bits.unique()):
+                sub_df = df[(df.backend == backend) &
+                            (df.base_model == base_model) &
+                            (df.gpu_count == gpu_count) &
+                            (df.bits == bits)].sort_values(by='gpu_name')
+                fig_bar.add_trace(go.Bar(x=sub_df.summary_out_throughput_normalize,
+                                         y=sub_df.gpu_name,
+                                         name=f'sum-{bits} bits',
+                                         legendgroup=f'sum-{bits} bits',
+                                         marker=dict(color=bits_bar_colors[f'{bits}']),
+                                         orientation='h'),
+                                  row=n_gpus.index(gpu_count) + 1,
+                                  col=base_models.index(base_model) * 2 + 1)
+                fig_bar.add_trace(go.Bar(x=sub_df.generate_out_throughput_normalize,
+                                         y=sub_df.gpu_name,
+                                         name=f'gen-{bits} bits',
+                                         legendgroup=f'gen-{bits} bits',
+                                         marker=dict(color=bits_bar_colors[f'{bits}']),
+                                         orientation='h'),
+                                  row=list(n_gpus).index(gpu_count) + 1,
+                                  col=list(base_models).index(base_model) * 2 + 2)
+
+    fig_bar.update_layout(plot_bgcolor='rgb(250,250,250)',
+                          showlegend=True,
+                          barmode="group")
+    # fig_bar.show()
+    fig_bar.write_html(f'llm_gpu_benchmark_{backend}.html', include_plotlyjs='cdn')

benchmarks/llm_gpu_benchmark_text-generation-inference.html

@@ -0,0 +1,7 @@
+<html>
+<head><meta charset="utf-8" /></head>
+<body>
+    <div>                        <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
+        <script src="https://cdn.plot.ly/plotly-2.2.0.min.js"></script>                <div id="8d98303e-9d8d-4a86-9ab9-85be1f565ba7" class="plotly-graph-div" style="height:100%; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("8d98303e-9d8d-4a86-9ab9-85be1f565ba7")) {                    Plotly.newPlot(                        "8d98303e-9d8d-4a86-9ab9-85be1f565ba7",                        [{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x","y":[],"yaxis":"y"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x2","y":[],"yaxis":"y2"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x","y":[],"yaxis":"y"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x2","y":[],"yaxis":"y2"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[31.964670378460696,40.07702972093452,28.212217062134258,24.76324507950772,29.383143217889106],"xaxis":"x","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[38.97113273835895,37.81293817302825,25.418311714688866,46.82453047975238,25.870047557539163],"xaxis":"x2","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y2"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x7","y":[],"yaxis":"y7"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x8","y":[],"yaxis":"y8"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x7","y":[],"yaxis":"y7"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x8","y":[],"yaxis":"y8"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[27.742149283479364,131.11372927692716,27.756812705358207],"xaxis":"x7","y":["45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y7"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[25.757641294033732,60.88036130542081,24.89894321470165],"xaxis":"x8","y":["45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y8"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x13","y":[],"yaxis":"y13"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x14","y":[],"yaxis":"y14"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x13","y":[],"yaxis":"y13"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x14","y":[],"yaxis":"y14"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[26.58192050074467,27.706125039541696],"xaxis":"x13","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y13"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[24.92264927072723,24.11901127583454],"xaxis":"x14","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y14"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x19","y":[],"yaxis":"y19"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x20","y":[],"yaxis":"y20"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x19","y":[],"yaxis":"y19"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x20","y":[],"yaxis":"y20"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[26.56845022740626],"xaxis":"x19","y":["80-A100 SXM4"],"yaxis":"y19"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[23.63055816163121],"xaxis":"x20","y":["80-A100 SXM4"],"yaxis":"y20"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x3","y":[],"yaxis":"y3"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x4","y":[],"yaxis":"y4"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x3","y":[],"yaxis":"y3"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x4","y":[],"yaxis":"y4"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[null,null,38.784585018023556,18.13337657657005],"xaxis":"x3","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y3"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[null,null,28.590730184060984,16.18347618092991],"xaxis":"x4","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y4"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x9","y":[],"yaxis":"y9"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x10","y":[],"yaxis":"y10"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x9","y":[],"yaxis":"y9"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x10","y":[],"yaxis":"y10"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[20.929693801547206,12.694114023867758,85.02391911717123,17.23203722663425],"xaxis":"x9","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y9"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[26.649908731325855,18.11013971401145,49.03779902422664,18.7070327239283],"xaxis":"x10","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y10"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x15","y":[],"yaxis":"y15"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x16","y":[],"yaxis":"y16"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x15","y":[],"yaxis":"y15"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x16","y":[],"yaxis":"y16"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[12.361580993407348,16.12018834278174],"xaxis":"x15","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y15"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[17.620036315851138,17.885323649884445],"xaxis":"x16","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y16"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x21","y":[],"yaxis":"y21"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x22","y":[],"yaxis":"y22"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x21","y":[],"yaxis":"y21"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x22","y":[],"yaxis":"y22"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[17.333509386436194],"xaxis":"x21","y":["80-A100 SXM4"],"yaxis":"y21"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[17.907476788430102],"xaxis":"x22","y":["80-A100 SXM4"],"yaxis":"y22"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x5","y":[],"yaxis":"y5"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x6","y":[],"yaxis":"y6"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x5","y":[],"yaxis":"y5"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x6","y":[],"yaxis":"y6"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x5","y":[],"yaxis":"y5"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x6","y":[],"yaxis":"y6"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x11","y":[],"yaxis":"y11"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x12","y":[],"yaxis":"y12"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x11","y":[],"yaxis":"y11"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x12","y":[],"yaxis":"y12"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x11","y":[],"yaxis":"y11"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x12","y":[],"yaxis":"y12"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x17","y":[],"yaxis":"y17"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x18","y":[],"yaxis":"y18"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x17","y":[],"yaxis":"y17"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x18","y":[],"yaxis":"y18"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[null,6.337898874140187],"xaxis":"x17","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y17"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[null,8.157040216950774],"xaxis":"x18","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y18"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x23","y":[],"yaxis":"y23"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x24","y":[],"yaxis":"y24"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x23","y":[],"yaxis":"y23"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x24","y":[],"yaxis":"y24"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[6.239297143818297],"xaxis":"x23","y":["80-A100 SXM4"],"yaxis":"y23"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[8.082069511295837],"xaxis":"x24","y":["80-A100 SXM4"],"yaxis":"y24"}],                        {"annotations":[{"font":{"size":16},"showarrow":false,"text":"llama2-7b-chat Summarization","x":0.06777777777777778,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-7b-chat Generation","x":0.2366666666666667,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-13b-chat Summarization","x":0.40555555555555556,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-13b-chat Generation","x":0.5744444444444445,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-70b-chat Summarization","x":0.7433333333333334,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-70b-chat Generation","x":0.9122222222222223,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"1 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.9125,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"2 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.6375000000000001,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"4 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.36250000000000004,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"8 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.0875,"yanchor":"middle","yref":"paper"}],"barmode":"group","plot_bgcolor":"rgb(250,250,250)","showlegend":true,"template":{"data":{"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"choropleth":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"choropleth"}],"contour":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"contour"}],"contourcarpet":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"contourcarpet"}],"heatmap":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"heatmap"}],"heatmapgl":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"heatmapgl"}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"histogram2d":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"histogram2d"}],"histogram2dcontour":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"histogram2dcontour"}],"mesh3d":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"mesh3d"}],"parcoords":[{"line":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"parcoords"}],"pie":[{"automargin":true,"type":"pie"}],"scatter":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatter"}],"scatter3d":[{"line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatter3d"}],"scattercarpet":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattercarpet"}],"scattergeo":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattergeo"}],"scattergl":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattergl"}],"scattermapbox":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattermapbox"}],"scatterpolar":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterpolar"}],"scatterpolargl":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterpolargl"}],"scatterternary":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterternary"}],"surface":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"surface"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}]},"layout":{"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"autotypenumbers":"strict","coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]],"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"geo":{"bgcolor":"white","lakecolor":"white","landcolor":"#E5ECF6","showlakes":true,"showland":true,"subunitcolor":"white"},"hoverlabel":{"align":"left"},"hovermode":"closest","mapbox":{"style":"light"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"bgcolor":"#E5ECF6","radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"ternary":{"aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"bgcolor":"#E5ECF6","caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"title":{"x":0.05},"xaxis":{"automargin":true,"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","zerolinewidth":2},"yaxis":{"automargin":true,"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","zerolinewidth":2}}},"xaxis":{"anchor":"y","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis10":{"anchor":"y10","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis11":{"anchor":"y11","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis12":{"anchor":"y12","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis13":{"anchor":"y13","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis14":{"anchor":"y14","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis15":{"anchor":"y15","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"xaxis16":{"anchor":"y16","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis17":{"anchor":"y17","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis18":{"anchor":"y18","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis19":{"anchor":"y19","domain":[0.0,0.13555555555555557]},"xaxis2":{"anchor":"y2","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis20":{"anchor":"y20","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19"},"xaxis21":{"anchor":"y21","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19"},"xaxis22":{"anchor":"y22","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19"},"xaxis23":{"anchor":"y23","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19"},"xaxis24":{"anchor":"y24","domain":[0.8444444444444444,0.98],"matches":"x19"},"xaxis3":{"anchor":"y3","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"xaxis4":{"anchor":"y4","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis5":{"anchor":"y5","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis6":{"anchor":"y6","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis7":{"anchor":"y7","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis8":{"anchor":"y8","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis9":{"anchor":"y9","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"yaxis":{"anchor":"x","domain":[0.825,1.0],"matches":"y19"},"yaxis10":{"anchor":"x10","domain":[0.55,0.7250000000000001],"matches":"y22"},"yaxis11":{"anchor":"x11","domain":[0.55,0.7250000000000001],"matches":"y23"},"yaxis12":{"anchor":"x12","domain":[0.55,0.7250000000000001],"matches":"y24"},"yaxis13":{"anchor":"x13","domain":[0.275,0.45],"matches":"y19"},"yaxis14":{"anchor":"x14","domain":[0.275,0.45],"matches":"y20"},"yaxis15":{"anchor":"x15","domain":[0.275,0.45],"matches":"y21"},"yaxis16":{"anchor":"x16","domain":[0.275,0.45],"matches":"y22"},"yaxis17":{"anchor":"x17","domain":[0.275,0.45],"matches":"y23"},"yaxis18":{"anchor":"x18","domain":[0.275,0.45],"matches":"y24"},"yaxis19":{"anchor":"x19","domain":[0.0,0.175]},"yaxis2":{"anchor":"x2","domain":[0.825,1.0],"matches":"y20"},"yaxis20":{"anchor":"x20","domain":[0.0,0.175]},"yaxis21":{"anchor":"x21","domain":[0.0,0.175]},"yaxis22":{"anchor":"x22","domain":[0.0,0.175]},"yaxis23":{"anchor":"x23","domain":[0.0,0.175]},"yaxis24":{"anchor":"x24","domain":[0.0,0.175]},"yaxis3":{"anchor":"x3","domain":[0.825,1.0],"matches":"y21"},"yaxis4":{"anchor":"x4","domain":[0.825,1.0],"matches":"y22"},"yaxis5":{"anchor":"x5","domain":[0.825,1.0],"matches":"y23"},"yaxis6":{"anchor":"x6","domain":[0.825,1.0],"matches":"y24"},"yaxis7":{"anchor":"x7","domain":[0.55,0.7250000000000001],"matches":"y19"},"yaxis8":{"anchor":"x8","domain":[0.55,0.7250000000000001],"matches":"y20"},"yaxis9":{"anchor":"x9","domain":[0.55,0.7250000000000001],"matches":"y21"}},                        {"responsive": true}                    )                };                            </script>        </div>
+</body>
+</html>

benchmarks/llm_gpu_benchmark_transformers.html

@@ -0,0 +1,7 @@
+<html>
+<head><meta charset="utf-8" /></head>
+<body>
+    <div>                        <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
+        <script src="https://cdn.plot.ly/plotly-2.2.0.min.js"></script>                <div id="4671500e-e030-484c-8d8f-02c9ef28c439" class="plotly-graph-div" style="height:100%; width:100%;"></div>            <script type="text/javascript">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById("4671500e-e030-484c-8d8f-02c9ef28c439")) {                    Plotly.newPlot(                        "4671500e-e030-484c-8d8f-02c9ef28c439",                        [{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[9.839381644193974,19.682153353799034,14.47651674912018,26.790154000919145,16.85058557689085],"xaxis":"x","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[12.67469844085007,27.622051912134882,19.374373797474846,27.42684895928983,20.2526752952322],"xaxis":"x2","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y2"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[4.417365201244467,11.290925144038532,6.08976919051411,9.56217317275004,5.9263976593415855],"xaxis":"x","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[3.695887145541112,7.812688672567852,5.614002693550519,7.59461596844275,6.252509885345299],"xaxis":"x2","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y2"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[18.73507454097704,39.43429532784967,27.07453064626594,39.96998450085984,29.3453161508673],"xaxis":"x","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[15.313436327725622,34.706856549443415,25.316661797353536,35.57028809081909,26.27458999671037],"xaxis":"x2","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y2"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[9.423935993931764,13.777794033942168,26.52473854898931,15.828182317775882],"xaxis":"x7","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y7"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[12.395401201017949,18.633481353508632,27.185836623669307,19.299187279602062],"xaxis":"x8","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y8"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[4.376286144153169,5.93295870509821,9.48124590639799,5.974715789431367],"xaxis":"x7","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y7"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[3.689148081304866,5.460311898298637,7.664435463393246,6.406802687346095],"xaxis":"x8","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y8"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[17.81624239176298,26.86157274268731,39.624799784757535,27.909081799152222],"xaxis":"x7","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y7"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[15.05454520400735,24.464037234597612,34.25052506253877,25.495156728837525],"xaxis":"x8","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y8"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[13.394795492541103,15.210707499507597],"xaxis":"x13","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y13"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[18.15606381072783,18.661753478727857],"xaxis":"x14","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y14"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[5.899421336969099,5.767145178389089],"xaxis":"x13","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y13"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[5.482425931352881,6.192523296540574],"xaxis":"x14","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y14"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[25.9430839554289,27.46244144955532],"xaxis":"x13","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y13"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[23.520372312313448,25.20924356998125],"xaxis":"x14","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y14"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[14.764927656045513],"xaxis":"x19","y":["80-A100 SXM4"],"yaxis":"y19"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[18.07719847124392],"xaxis":"x20","y":["80-A100 SXM4"],"yaxis":"y20"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[5.718961706449293],"xaxis":"x19","y":["80-A100 SXM4"],"yaxis":"y19"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[6.177879854004683],"xaxis":"x20","y":["80-A100 SXM4"],"yaxis":"y20"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[27.054106396318144],"xaxis":"x19","y":["80-A100 SXM4"],"yaxis":"y19"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[25.138719102309768],"xaxis":"x20","y":["80-A100 SXM4"],"yaxis":"y20"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[8.083390907285379,12.174340676118161,11.076606608131389,16.98095523506584,12.1008725506651],"xaxis":"x3","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y3"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[8.287678373962581,13.674114390829141,13.308822531004934,17.365713991091738,12.794482361704157],"xaxis":"x4","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y4"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[3.1403647823510736,4.962801741500335,3.5348819482865093,5.3562909858984185,4.1213135763128905],"xaxis":"x3","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y3"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[3.18815245154689,5.105728547922034,4.718240806380357,6.509024089959697,4.827719089783637],"xaxis":"x4","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y4"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[null,null,11.435609485285738,17.906931325335666,18.878279411581737],"xaxis":"x3","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y3"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[null,null,15.63493452970772,22.260343102292754,21.142120495293863],"xaxis":"x4","y":["24-RTX 3090","24-RTX 4090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y4"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[7.931395602652238,10.626258179366356,16.319110879759947,11.241866660596408],"xaxis":"x9","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y9"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[8.215795602873966,12.833338647314658,16.943446615015436,12.165800832662722],"xaxis":"x10","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y10"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[3.1362935762237645,3.499943275803895,5.402452917863267,3.9771491776646073],"xaxis":"x9","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y9"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[3.156100924190738,4.674808411970743,6.638529207897594,4.611620121814299],"xaxis":"x10","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y10"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[9.178059918412773,11.287265701494618,18.203631997182082,18.325614335569053],"xaxis":"x9","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y9"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[10.880162241524287,15.437944210820223,22.20571335065674,21.096027375985646],"xaxis":"x10","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y10"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[10.252856056970655,11.390035634842294],"xaxis":"x15","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y15"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[12.44766998737035,12.445574043628245],"xaxis":"x16","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y16"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[3.432161213004653,4.005435712274412],"xaxis":"x15","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y15"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[4.577172738204334,4.623478053690466],"xaxis":"x16","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y16"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[11.047490604822276,16.97583795634349],"xaxis":"x15","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y15"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[15.140929085583872,18.40904684710705],"xaxis":"x16","y":["45-RTX A6000","80-A100 SXM4"],"yaxis":"y16"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[10.82198892665345],"xaxis":"x21","y":["80-A100 SXM4"],"yaxis":"y21"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[11.846523539191672],"xaxis":"x22","y":["80-A100 SXM4"],"yaxis":"y22"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[3.8795801184687786],"xaxis":"x21","y":["80-A100 SXM4"],"yaxis":"y21"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[4.568029810459134],"xaxis":"x22","y":["80-A100 SXM4"],"yaxis":"y22"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[16.97013525520682],"xaxis":"x21","y":["80-A100 SXM4"],"yaxis":"y21"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[18.913362098572737],"xaxis":"x22","y":["80-A100 SXM4"],"yaxis":"y22"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[null,5.161890396610965,6.976123395155549],"xaxis":"x5","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y5"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[null,5.887611768925055,9.031399021823733],"xaxis":"x6","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y6"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[null,null,null],"xaxis":"x5","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y5"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[null,null,null],"xaxis":"x6","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y6"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[null,null,null,null],"xaxis":"x5","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y5"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[null,null,null,null],"xaxis":"x6","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation","80-A100 SXM4"],"yaxis":"y6"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[null,4.96092701086689,7.068376492905629],"xaxis":"x11","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y11"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[null,5.693355665703394,8.905280446876153],"xaxis":"x12","y":["24-RTX 3090","45-RTX A6000","48-RTX 6000 Ada Generation"],"yaxis":"y12"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[1.9856691832414866],"xaxis":"x11","y":["45-RTX A6000"],"yaxis":"y11"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[1.913951722547195],"xaxis":"x12","y":["45-RTX A6000"],"yaxis":"y12"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[null],"xaxis":"x11","y":["45-RTX A6000"],"yaxis":"y11"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[null],"xaxis":"x12","y":["45-RTX A6000"],"yaxis":"y12"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[4.8550061015042685],"xaxis":"x17","y":["45-RTX A6000"],"yaxis":"y17"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[5.58004075989967],"xaxis":"x18","y":["45-RTX A6000"],"yaxis":"y18"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[1.9670200139619358],"xaxis":"x17","y":["45-RTX A6000"],"yaxis":"y17"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[1.8873606277914459],"xaxis":"x18","y":["45-RTX A6000"],"yaxis":"y18"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[2.665381007576966],"xaxis":"x17","y":["45-RTX A6000"],"yaxis":"y17"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[3.597816249219273],"xaxis":"x18","y":["45-RTX A6000"],"yaxis":"y18"},{"legendgroup":"sum-4 bits","marker":{"color":"#1F77B4"},"name":"sum-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x23","y":[],"yaxis":"y23"},{"legendgroup":"gen-4 bits","marker":{"color":"#1F77B4"},"name":"gen-4 bits","orientation":"h","type":"bar","x":[],"xaxis":"x24","y":[],"yaxis":"y24"},{"legendgroup":"sum-8 bits","marker":{"color":"#FF7F0E"},"name":"sum-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x23","y":[],"yaxis":"y23"},{"legendgroup":"gen-8 bits","marker":{"color":"#FF7F0E"},"name":"gen-8 bits","orientation":"h","type":"bar","x":[],"xaxis":"x24","y":[],"yaxis":"y24"},{"legendgroup":"sum-16 bits","marker":{"color":"#2CA02C"},"name":"sum-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x23","y":[],"yaxis":"y23"},{"legendgroup":"gen-16 bits","marker":{"color":"#2CA02C"},"name":"gen-16 bits","orientation":"h","type":"bar","x":[],"xaxis":"x24","y":[],"yaxis":"y24"}],                        {"annotations":[{"font":{"size":16},"showarrow":false,"text":"llama2-7b-chat Summarization","x":0.06777777777777778,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-7b-chat Generation","x":0.2366666666666667,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-13b-chat Summarization","x":0.40555555555555556,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-13b-chat Generation","x":0.5744444444444445,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-70b-chat Summarization","x":0.7433333333333334,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"llama2-70b-chat Generation","x":0.9122222222222223,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"1 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.9125,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"2 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.6375000000000001,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"4 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.36250000000000004,"yanchor":"middle","yref":"paper"},{"font":{"size":16},"showarrow":false,"text":"8 GPUs","textangle":90,"x":0.98,"xanchor":"left","xref":"paper","y":0.0875,"yanchor":"middle","yref":"paper"}],"barmode":"group","plot_bgcolor":"rgb(250,250,250)","showlegend":true,"template":{"data":{"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"barpolar":[{"marker":{"line":{"color":"#E5ECF6","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"white","linecolor":"white","minorgridcolor":"white","startlinecolor":"#2a3f5f"},"type":"carpet"}],"choropleth":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"choropleth"}],"contour":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"contour"}],"contourcarpet":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"contourcarpet"}],"heatmap":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"heatmap"}],"heatmapgl":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"heatmapgl"}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"histogram2d":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"histogram2d"}],"histogram2dcontour":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"histogram2dcontour"}],"mesh3d":[{"colorbar":{"outlinewidth":0,"ticks":""},"type":"mesh3d"}],"parcoords":[{"line":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"parcoords"}],"pie":[{"automargin":true,"type":"pie"}],"scatter":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatter"}],"scatter3d":[{"line":{"colorbar":{"outlinewidth":0,"ticks":""}},"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatter3d"}],"scattercarpet":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattercarpet"}],"scattergeo":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattergeo"}],"scattergl":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattergl"}],"scattermapbox":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scattermapbox"}],"scatterpolar":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterpolar"}],"scatterpolargl":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterpolargl"}],"scatterternary":[{"marker":{"colorbar":{"outlinewidth":0,"ticks":""}},"type":"scatterternary"}],"surface":[{"colorbar":{"outlinewidth":0,"ticks":""},"colorscale":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"type":"surface"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}]},"layout":{"annotationdefaults":{"arrowcolor":"#2a3f5f","arrowhead":0,"arrowwidth":1},"autotypenumbers":"strict","coloraxis":{"colorbar":{"outlinewidth":0,"ticks":""}},"colorscale":{"diverging":[[0,"#8e0152"],[0.1,"#c51b7d"],[0.2,"#de77ae"],[0.3,"#f1b6da"],[0.4,"#fde0ef"],[0.5,"#f7f7f7"],[0.6,"#e6f5d0"],[0.7,"#b8e186"],[0.8,"#7fbc41"],[0.9,"#4d9221"],[1,"#276419"]],"sequential":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]],"sequentialminus":[[0.0,"#0d0887"],[0.1111111111111111,"#46039f"],[0.2222222222222222,"#7201a8"],[0.3333333333333333,"#9c179e"],[0.4444444444444444,"#bd3786"],[0.5555555555555556,"#d8576b"],[0.6666666666666666,"#ed7953"],[0.7777777777777778,"#fb9f3a"],[0.8888888888888888,"#fdca26"],[1.0,"#f0f921"]]},"colorway":["#636efa","#EF553B","#00cc96","#ab63fa","#FFA15A","#19d3f3","#FF6692","#B6E880","#FF97FF","#FECB52"],"font":{"color":"#2a3f5f"},"geo":{"bgcolor":"white","lakecolor":"white","landcolor":"#E5ECF6","showlakes":true,"showland":true,"subunitcolor":"white"},"hoverlabel":{"align":"left"},"hovermode":"closest","mapbox":{"style":"light"},"paper_bgcolor":"white","plot_bgcolor":"#E5ECF6","polar":{"angularaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"bgcolor":"#E5ECF6","radialaxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"scene":{"xaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"},"yaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"},"zaxis":{"backgroundcolor":"#E5ECF6","gridcolor":"white","gridwidth":2,"linecolor":"white","showbackground":true,"ticks":"","zerolinecolor":"white"}},"shapedefaults":{"line":{"color":"#2a3f5f"}},"ternary":{"aaxis":{"gridcolor":"white","linecolor":"white","ticks":""},"baxis":{"gridcolor":"white","linecolor":"white","ticks":""},"bgcolor":"#E5ECF6","caxis":{"gridcolor":"white","linecolor":"white","ticks":""}},"title":{"x":0.05},"xaxis":{"automargin":true,"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","zerolinewidth":2},"yaxis":{"automargin":true,"gridcolor":"white","linecolor":"white","ticks":"","title":{"standoff":15},"zerolinecolor":"white","zerolinewidth":2}}},"xaxis":{"anchor":"y","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis10":{"anchor":"y10","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis11":{"anchor":"y11","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis12":{"anchor":"y12","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis13":{"anchor":"y13","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis14":{"anchor":"y14","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis15":{"anchor":"y15","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"xaxis16":{"anchor":"y16","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis17":{"anchor":"y17","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis18":{"anchor":"y18","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis19":{"anchor":"y19","domain":[0.0,0.13555555555555557]},"xaxis2":{"anchor":"y2","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis20":{"anchor":"y20","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19"},"xaxis21":{"anchor":"y21","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19"},"xaxis22":{"anchor":"y22","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19"},"xaxis23":{"anchor":"y23","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19"},"xaxis24":{"anchor":"y24","domain":[0.8444444444444444,0.98],"matches":"x19"},"xaxis3":{"anchor":"y3","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"xaxis4":{"anchor":"y4","domain":[0.5066666666666667,0.6422222222222222],"matches":"x19","showticklabels":false},"xaxis5":{"anchor":"y5","domain":[0.6755555555555556,0.8111111111111111],"matches":"x19","showticklabels":false},"xaxis6":{"anchor":"y6","domain":[0.8444444444444444,0.98],"matches":"x19","showticklabels":false},"xaxis7":{"anchor":"y7","domain":[0.0,0.13555555555555557],"matches":"x19","showticklabels":false},"xaxis8":{"anchor":"y8","domain":[0.1688888888888889,0.30444444444444446],"matches":"x19","showticklabels":false},"xaxis9":{"anchor":"y9","domain":[0.3377777777777778,0.4733333333333334],"matches":"x19","showticklabels":false},"yaxis":{"anchor":"x","domain":[0.825,1.0],"matches":"y19"},"yaxis10":{"anchor":"x10","domain":[0.55,0.7250000000000001],"matches":"y22"},"yaxis11":{"anchor":"x11","domain":[0.55,0.7250000000000001],"matches":"y23"},"yaxis12":{"anchor":"x12","domain":[0.55,0.7250000000000001],"matches":"y24"},"yaxis13":{"anchor":"x13","domain":[0.275,0.45],"matches":"y19"},"yaxis14":{"anchor":"x14","domain":[0.275,0.45],"matches":"y20"},"yaxis15":{"anchor":"x15","domain":[0.275,0.45],"matches":"y21"},"yaxis16":{"anchor":"x16","domain":[0.275,0.45],"matches":"y22"},"yaxis17":{"anchor":"x17","domain":[0.275,0.45],"matches":"y23"},"yaxis18":{"anchor":"x18","domain":[0.275,0.45],"matches":"y24"},"yaxis19":{"anchor":"x19","domain":[0.0,0.175]},"yaxis2":{"anchor":"x2","domain":[0.825,1.0],"matches":"y20"},"yaxis20":{"anchor":"x20","domain":[0.0,0.175]},"yaxis21":{"anchor":"x21","domain":[0.0,0.175]},"yaxis22":{"anchor":"x22","domain":[0.0,0.175]},"yaxis23":{"anchor":"x23","domain":[0.0,0.175]},"yaxis24":{"anchor":"x24","domain":[0.0,0.175]},"yaxis3":{"anchor":"x3","domain":[0.825,1.0],"matches":"y21"},"yaxis4":{"anchor":"x4","domain":[0.825,1.0],"matches":"y22"},"yaxis5":{"anchor":"x5","domain":[0.825,1.0],"matches":"y23"},"yaxis6":{"anchor":"x6","domain":[0.825,1.0],"matches":"y24"},"yaxis7":{"anchor":"x7","domain":[0.55,0.7250000000000001],"matches":"y19"},"yaxis8":{"anchor":"x8","domain":[0.55,0.7250000000000001],"matches":"y20"},"yaxis9":{"anchor":"x9","domain":[0.55,0.7250000000000001],"matches":"y21"}},                        {"responsive": true}                    )                };                            </script>        </div>
+</body>
+</html>

benchmarks/llm_gpu_benchmarks.json

@@ -0,0 +1,2790 @@
+[
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 10:46:19",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 32.29472152392069,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 14.563165505727133
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 10:48:55",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 67.97515447934468,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 33.00641902287801
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 10:48:58",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1440,
+    "summarize_time": 114.62220064798991,
+    "generate_output_len_bytes": 2619,
+    "generate_time": 71.0722058614095
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 10:58:34",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 866,
+    "summarize_time": 39.54404203097025,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 22.466302394866943
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:01:59",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 32.1394579410553,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 14.757195552190145
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 10:54:29",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 910,
+    "summarize_time": 185.14580019315085,
+    "generate_output_len_bytes": 2042,
+    "generate_time": 117.13909141222636
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:04:37",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 94.98129558563232,
+    "generate_output_len_bytes": 2512,
+    "generate_time": 69.4871145884196
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:13:08",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1276,
+    "summarize_time": 43.23498781522115,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 22.826789538065594
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 11:10:08",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 991,
+    "summarize_time": 90.51939169565837,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 48.96095744768778
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 11:16:48",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 31.86189842224121,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 14.209659894307455
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:17:39",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 71.48081835110982,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 33.5740262667338
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 11:19:24",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 94.17744310696919,
+    "generate_output_len_bytes": 2512,
+    "generate_time": 70.12592967351277
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 11:27:57",
+    "git_sha": "55d3b55b",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1276,
+    "summarize_time": 42.8066500822703,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 22.626200040181477
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:23:22",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 910,
+    "summarize_time": 186.88371555010477,
+    "generate_output_len_bytes": 2042,
+    "generate_time": 117.3530724843343
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 11:39:03",
+    "git_sha": "55d3b55b",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 991,
+    "summarize_time": 94.50985678037007,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 50.06416177749634
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 21:08:31",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 38.80374129613241,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 19.23690136273702
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 21:11:49",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 178.79640992482504,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 93.99476226170857
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 21:25:53",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 53.44271365801493,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 30.641155401865642
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 21:30:30",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 40.80062770843506,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 19.825008392333984
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 21:35:29",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 177.35046529769897,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 91.73111907641093
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 21:49:20",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 56.894784371058144,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 32.15500020980835
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/18/2023 21:54:11",
+    "git_sha": "fc4826f2",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 41.46419604619344,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 20.049855709075928
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/18/2023 21:57:39",
+    "git_sha": "fc4826f2",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 183.73364853858948,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 94.9052836894989
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/18/2023 22:11:59",
+    "git_sha": "fc4826f2",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 59.204413731892906,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 33.25332593917847
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/18/2023 22:17:00",
+    "git_sha": "fc4826f2",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 42.09002653757731,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 20.106103817621868
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/18/2023 22:20:31",
+    "git_sha": "fc4826f2",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 185.28164370854697,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 95.13023789723714
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/18/2023 22:34:58",
+    "git_sha": "fc4826f2",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 60.9919019540151,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 34.328625202178955
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:31:34",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 52.49842747052511,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 20.686774571736652
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:31:55",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:35:38",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1007,
+    "summarize_time": 168.9666860898336,
+    "generate_output_len_bytes": 2249,
+    "generate_time": 73.25518870353699
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:48:09",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 856,
+    "summarize_time": 45.30513469378153,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 22.000216643015545
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 13:51:56",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 51.64275654157003,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 20.737667481104534
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:35:47",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 980,
+    "summarize_time": 280.4669913450877,
+    "generate_output_len_bytes": 2132,
+    "generate_time": 141.7793349424998
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 13:57:35",
+    "git_sha": "fc4826f2",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 869,
+    "summarize_time": 96.61887431144714,
+    "generate_output_len_bytes": 3244,
+    "generate_time": 82.98751719792683
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 13:55:51",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1007,
+    "summarize_time": 167.52292919158936,
+    "generate_output_len_bytes": 2249,
+    "generate_time": 71.82611886660258
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 14:08:08",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 856,
+    "summarize_time": 47.14254776636759,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 22.54850967725118
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 14:15:15",
+    "git_sha": "d13230ee",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 14:07:15",
+    "git_sha": "fc4826f2",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 89.59958203633626,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 42.32424934705099
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 14:15:30",
+    "git_sha": "d13230ee",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1024,
+    "summarize_time": 185.44230167071024,
+    "generate_output_len_bytes": 2122,
+    "generate_time": 88.11553311347961
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 14:29:36",
+    "git_sha": "d13230ee",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 922,
+    "summarize_time": 68.06459252039592,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 27.939613421758015
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 14:26:29",
+    "git_sha": "d13230ee",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 980,
+    "summarize_time": 280.8310640652974,
+    "generate_output_len_bytes": 2132,
+    "generate_time": 143.21916349728903
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 14:48:17",
+    "git_sha": "d13230ee",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 869,
+    "summarize_time": 98.47045453389485,
+    "generate_output_len_bytes": 3244,
+    "generate_time": 83.71360301971436
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 15:35:13",
+    "git_sha": "0dec0f52",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 15:49:33",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 16:26:53",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 16:27:32",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 16:29:03",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 17:26:02",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 18:59:16",
+    "git_sha": "5691db4a",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1075,
+    "summarize_time": 39.01545596122742,
+    "generate_output_len_bytes": 2242,
+    "generate_time": 10.151424566904703
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 19:03:13",
+    "git_sha": "5691db4a",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 940,
+    "summarize_time": 21.78233750661214,
+    "generate_output_len_bytes": 2130,
+    "generate_time": 15.794983307520548
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 19:38:40",
+    "git_sha": "6f05e8f1",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1114,
+    "summarize_time": 7.636120955149333,
+    "generate_output_len_bytes": 2275,
+    "generate_time": 7.922623078028361
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 19:41:02",
+    "git_sha": "6f05e8f1",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1024,
+    "summarize_time": 10.824170271555582,
+    "generate_output_len_bytes": 2130,
+    "generate_time": 9.209020694096884
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 19:55:17",
+    "git_sha": "2c548f21",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1088,
+    "summarize_time": 24.39883820215861,
+    "generate_output_len_bytes": 2275,
+    "generate_time": 12.755743900934855
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 00:57:21",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 37.113919814427696,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 18.36507821083069
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:00:31",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 49.79721482594808,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 21.780913591384888
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:04:36",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:05:26",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 181.2461258570353,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 92.64811905225118
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:19:33",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 800,
+    "summarize_time": 174.4576851526896,
+    "generate_output_len_bytes": 2713,
+    "generate_time": 119.14412077267964
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:36:14",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 53.39731526374817,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 31.369641542434692
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:40:53",
+    "git_sha": "a227be4f",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 74.27096923192342,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 29.860486666361492
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 01:48:09",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 39.926851193110146,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 18.481745958328247
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 01:51:27",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 51.299002488454185,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 21.828503131866455
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 01:56:20",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 178.19972308476767,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 91.73426882425944
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 02:10:13",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 800,
+    "summarize_time": 180.7814578215281,
+    "generate_output_len_bytes": 2713,
+    "generate_time": 124.72717420260112
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 02:26:43",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 57.08081785837809,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 32.26534946759542
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 02:31:36",
+    "git_sha": "a227be4f",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 79.9461121559143,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 31.403561115264893
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 02:38:23",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 42.33977222442627,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 19.723278522491455
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 02:41:52",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 55.377869288126625,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 25.01458676656087
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 02:47:05",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 180.53432401021323,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 91.93375285466512
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:01:07",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 800,
+    "summarize_time": 179.50477250417075,
+    "generate_output_len_bytes": 2713,
+    "generate_time": 124.40728378295898
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:17:36",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 58.62867816289266,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 33.394495725631714
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:22:37",
+    "git_sha": "a227be4f",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 78.90612125396729,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 30.697617371877033
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 03:29:20",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 40.498607873916626,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 19.509677171707153
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 03:32:44",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 55.3964786529541,
+    "generate_output_len_bytes": 2172,
+    "generate_time": 24.347585439682007
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 03:37:55",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1179,
+    "summarize_time": 186.71331850687662,
+    "generate_output_len_bytes": 2772,
+    "generate_time": 95.784650405248
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 03:52:28",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 800,
+    "summarize_time": 185.3280005455017,
+    "generate_output_len_bytes": 2713,
+    "generate_time": 125.91738017400105
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 04:09:18",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1002,
+    "summarize_time": 60.18280680974325,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 33.386961142222084
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/19/2023 04:14:25",
+    "git_sha": "a227be4f",
+    "n_gpus": 8,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 83.04790727297465,
+    "generate_output_len_bytes": 1802,
+    "generate_time": 32.24992283185323
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 23:26:19",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 47.03754989306132,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 19.964784463246662
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 23:33:09",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 71.91136892636617,
+    "generate_output_len_bytes": 2480,
+    "generate_time": 33.6295014222463
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 23:44:08",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 00:45:42",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1007,
+    "summarize_time": 148.61560583114624,
+    "generate_output_len_bytes": 2357,
+    "generate_time": 89.01266026496887
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 00:58:00",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 763,
+    "summarize_time": 193.99270629882812,
+    "generate_output_len_bytes": 2129,
+    "generate_time": 95.66660761833191
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:13:01",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:13:55",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 991,
+    "summarize_time": 61.52411222457886,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 32.030215660730995
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 01:19:00",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 1,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 81.13888708750407,
+    "generate_output_len_bytes": 3486,
+    "generate_time": 55.5331826210022
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 01:27:49",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 47.41046245892843,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 20.660600344340008
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 01:34:28",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 72.85646979014079,
+    "generate_output_len_bytes": 2480,
+    "generate_time": 34.05861854553223
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 02:39:22",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1007,
+    "summarize_time": 152.54357608159384,
+    "generate_output_len_bytes": 2357,
+    "generate_time": 91.51808977127075
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 02:52:58",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 763,
+    "summarize_time": 195.92926557858786,
+    "generate_output_len_bytes": 2129,
+    "generate_time": 96.55542047818501
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 03:15:01",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 991,
+    "summarize_time": 64.64422671000163,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 33.30378039677938
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 03:20:19",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 2,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 84.57761120796204,
+    "generate_output_len_bytes": 3486,
+    "generate_time": 57.59072462717692
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:28:44",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1417,
+    "summarize_time": 49.08898218472799,
+    "generate_output_len_bytes": 2384,
+    "generate_time": 21.489527861277264
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:32:39",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 74.43774898846944,
+    "generate_output_len_bytes": 2480,
+    "generate_time": 34.72673638661703
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:39:21",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1007,
+    "summarize_time": 153.41076453526816,
+    "generate_output_len_bytes": 2357,
+    "generate_time": 91.14894040425618
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 03:52:00",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 763,
+    "summarize_time": 199.79869039853415,
+    "generate_output_len_bytes": 2129,
+    "generate_time": 98.61504419644673
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 04:08:12",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 991,
+    "summarize_time": 66.49260465304057,
+    "generate_output_len_bytes": 2927,
+    "generate_time": 34.17951035499573
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 04:13:39",
+    "git_sha": "0cdb75ef",
+    "n_gpus": 4,
+    "transformers": "4.30.2",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1000,
+    "summarize_time": 87.65787092844646,
+    "generate_output_len_bytes": 3486,
+    "generate_time": 59.3750696182251
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 22:22:24",
+    "git_sha": "b63768c6",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 948,
+    "summarize_time": 122.13213857014973,
+    "generate_output_len_bytes": 2826,
+    "generate_time": 66.34098903338115
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/18/2023 22:33:33",
+    "git_sha": "c1348fb3",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 948,
+    "summarize_time": 120.53812781969707,
+    "generate_output_len_bytes": 2826,
+    "generate_time": 67.28052496910095
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 22:56:52",
+    "git_sha": "fb84de76",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1036,
+    "summarize_time": 29.128981749216717,
+    "generate_output_len_bytes": 2242,
+    "generate_time": 12.197122732798258
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/18/2023 23:00:33",
+    "git_sha": "fb84de76",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 05:47:43",
+    "git_sha": "22352acd",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 05:48:58",
+    "git_sha": "22352acd",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/19/2023 05:50:40",
+    "git_sha": "22352acd",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 948,
+    "summarize_time": 165.05752809842429,
+    "generate_output_len_bytes": 2605,
+    "generate_time": 93.80659619967143
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 06:05:51",
+    "git_sha": "22352acd",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 06:10:05",
+    "git_sha": "22352acd",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 906,
+    "summarize_time": 410.0691332022349,
+    "generate_output_len_bytes": 521,
+    "generate_time": 57.71272214253744
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 06:36:58",
+    "git_sha": "22352acd",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 948,
+    "summarize_time": 171.74388321240744,
+    "generate_output_len_bytes": 2605,
+    "generate_time": 97.00725762049358
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 06:51:13",
+    "git_sha": "22352acd",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 792,
+    "summarize_time": 267.0555826822917,
+    "generate_output_len_bytes": 2783,
+    "generate_time": 163.99818523724875
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 07:13:35",
+    "git_sha": "22352acd",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 906,
+    "summarize_time": 413.9569679101308,
+    "generate_output_len_bytes": 521,
+    "generate_time": 58.52583885192871
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/19/2023 07:38:02",
+    "git_sha": "22352acd",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 948,
+    "summarize_time": 175.4907926718394,
+    "generate_output_len_bytes": 2605,
+    "generate_time": 98.97720170021057
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/19/2023 12:35:08",
+    "git_sha": "29a002e5",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "timemachine",
+    "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 983,
+    "summarize_time": 42.21107586224874,
+    "generate_output_len_bytes": 2130,
+    "generate_time": 16.94527777036031
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/21/2023 20:03:36",
+    "git_sha": "51318f44",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 41.0461368560791,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 19.614749511082966
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 20:07:35",
+    "git_sha": "51318f44",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 42.8376894791921,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 20.2719091574351
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/21/2023 20:42:46",
+    "git_sha": "2f4bb620",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 20:50:19",
+    "git_sha": "2f4bb620",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 66.52468911806743,
+    "generate_output_len_bytes": 2479,
+    "generate_time": 29.828714847564697
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 20:56:04",
+    "git_sha": "2f4bb620",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)",
+    "exception": "OOM"
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/21/2023 19:55:35",
+    "git_sha": "51318f44",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 38.753786404927574,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 19.529522736867268
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/21/2023 20:36:13",
+    "git_sha": "51318f44",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 41.024452924728394,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 20.29120985666911
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/21/2023 20:40:08",
+    "git_sha": "51318f44",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 54.554532527923584,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 24.604793945948284
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 20:50:05",
+    "git_sha": "51318f44",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 41.09950613975525,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 20.947362899780273
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 20:54:08",
+    "git_sha": "51318f44",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 58.3172922929128,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 25.735217014948528
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/21/2023 21:01:04",
+    "git_sha": "51318f44",
+    "n_gpus": 8,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 42.85940829912821,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 21.380353291829426
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/21/2023 21:05:24",
+    "git_sha": "51318f44",
+    "n_gpus": 8,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 54.235164642333984,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 25.70338026682536
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/21/2023 21:10:37",
+    "git_sha": "51318f44",
+    "n_gpus": 8,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 927,
+    "summarize_time": 133.53030570348105,
+    "generate_output_len_bytes": 2782,
+    "generate_time": 72.97924383481343
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 22:18:17",
+    "git_sha": "51318f44",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 927,
+    "summarize_time": 131.45291074117026,
+    "generate_output_len_bytes": 2782,
+    "generate_time": 72.30849742889404
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/21/2023 22:51:09",
+    "git_sha": "383b6bbc",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 39.269713958104454,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 19.65731406211853
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/21/2023 22:54:54",
+    "git_sha": "383b6bbc",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 51.84283971786499,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 28.441521485646565
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/21/2023 23:13:10",
+    "git_sha": "383b6bbc",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 53.383726040522255,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 24.422890504201252
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 4,
+    "reps": 3,
+    "date": "08/21/2023 23:18:04",
+    "git_sha": "383b6bbc",
+    "n_gpus": 4,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 52.791220347086586,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 25.378511508305866
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 8,
+    "reps": 3,
+    "date": "08/21/2023 23:23:11",
+    "git_sha": "383b6bbc",
+    "n_gpus": 8,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.8",
+    "hostname": "cloudvm",
+    "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1046,
+    "summarize_time": 56.3846542040507,
+    "generate_output_len_bytes": 2171,
+    "generate_time": 26.636192480723064
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 1,
+    "reps": 3,
+    "date": "08/21/2023 23:52:44",
+    "git_sha": "da69b822",
+    "n_gpus": 1,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1267,
+    "summarize_time": 40.36223220825195,
+    "generate_output_len_bytes": 2383,
+    "generate_time": 19.87660264968872
+  },
+  {
+    "backend": "text-generation-inference",
+    "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 2,
+    "reps": 3,
+    "date": "08/22/2023 00:15:05",
+    "git_sha": "e843e8c3",
+    "n_gpus": 2,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "recypabaszmhhmuae",
+    "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 915,
+    "summarize_time": 64.78201874097188,
+    "generate_output_len_bytes": 2479,
+    "generate_time": 29.02147897084554
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 16,
+    "ngpus": 0,
+    "reps": 3,
+    "date": "08/22/2023 19:01:15",
+    "git_sha": "855b7d15",
+    "n_gpus": 0,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "CPU",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1351,
+    "summarize_time": 1215.5185990333557,
+    "generate_output_len_bytes": 849,
+    "generate_time": 180.56836318969727
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 8,
+    "ngpus": 0,
+    "reps": 3,
+    "date": "08/22/2023 20:11:16",
+    "git_sha": "855b7d15",
+    "n_gpus": 0,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "CPU",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1353,
+    "summarize_time": 1216.9783231417339,
+    "generate_output_len_bytes": 849,
+    "generate_time": 180.42225472132364
+  },
+  {
+    "backend": "transformers",
+    "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat",
+    "task": "summary_and_generate",
+    "bits": 4,
+    "ngpus": 0,
+    "reps": 3,
+    "date": "08/22/2023 21:21:20",
+    "git_sha": "855b7d15",
+    "n_gpus": 0,
+    "transformers": "4.31.0",
+    "bitsandbytes": "0.41.1",
+    "cuda": "11.7",
+    "hostname": "rippa",
+    "gpus": "CPU",
+    "summarize_input_len_bytes": 857252,
+    "summarize_output_len_bytes": 1354,
+    "summarize_time": 1217.1687794526417,
+    "generate_output_len_bytes": 843,
+    "generate_time": 180.78463260332742
+  }
+]

benchmarks/perf.json

@@ -0,0 +1,136 @@
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 10:46:19", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 32.29472152392069, "generate_output_len_bytes": 2384, "generate_time": 14.563165505727133}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 10:48:55", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 67.97515447934468, "generate_output_len_bytes": 2384, "generate_time": 33.00641902287801}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 10:48:58", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1440, "summarize_time": 114.62220064798991, "generate_output_len_bytes": 2619, "generate_time": 71.0722058614095}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 10:58:34", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 866, "summarize_time": 39.54404203097025, "generate_output_len_bytes": 2927, "generate_time": 22.466302394866943}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:01:59", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 32.1394579410553, "generate_output_len_bytes": 2384, "generate_time": 14.757195552190145}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 10:54:29", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 910, "summarize_time": 185.14580019315085, "generate_output_len_bytes": 2042, "generate_time": 117.13909141222636}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:04:37", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 94.98129558563232, "generate_output_len_bytes": 2512, "generate_time": 69.4871145884196}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:13:08", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1276, "summarize_time": 43.23498781522115, "generate_output_len_bytes": 2927, "generate_time": 22.826789538065594}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 11:10:08", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 991, "summarize_time": 90.51939169565837, "generate_output_len_bytes": 2927, "generate_time": 48.96095744768778}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 11:16:48", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 31.86189842224121, "generate_output_len_bytes": 2384, "generate_time": 14.209659894307455}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:17:39", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 71.48081835110982, "generate_output_len_bytes": 2384, "generate_time": 33.5740262667338}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 11:19:24", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 94.17744310696919, "generate_output_len_bytes": 2512, "generate_time": 70.12592967351277}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 11:27:57", "git_sha": "55d3b55b", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1276, "summarize_time": 42.8066500822703, "generate_output_len_bytes": 2927, "generate_time": 22.626200040181477}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:23:22", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 910, "summarize_time": 186.88371555010477, "generate_output_len_bytes": 2042, "generate_time": 117.3530724843343}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 11:39:03", "git_sha": "55d3b55b", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 991, "summarize_time": 94.50985678037007, "generate_output_len_bytes": 2927, "generate_time": 50.06416177749634}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 21:08:31", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 38.80374129613241, "generate_output_len_bytes": 2384, "generate_time": 19.23690136273702}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 21:11:49", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 178.79640992482504, "generate_output_len_bytes": 2772, "generate_time": 93.99476226170857}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 21:25:53", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 53.44271365801493, "generate_output_len_bytes": 2927, "generate_time": 30.641155401865642}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 21:30:30", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 40.80062770843506, "generate_output_len_bytes": 2384, "generate_time": 19.825008392333984}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/18/2023 21:35:29", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 177.35046529769897, "generate_output_len_bytes": 2772, "generate_time": 91.73111907641093}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 21:49:20", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 56.894784371058144, "generate_output_len_bytes": 2927, "generate_time": 32.15500020980835}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/18/2023 21:54:11", "git_sha": "fc4826f2", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 41.46419604619344, "generate_output_len_bytes": 2384, "generate_time": 20.049855709075928}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/18/2023 21:57:39", "git_sha": "fc4826f2", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 183.73364853858948, "generate_output_len_bytes": 2772, "generate_time": 94.9052836894989}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/18/2023 22:11:59", "git_sha": "fc4826f2", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 59.204413731892906, "generate_output_len_bytes": 2927, "generate_time": 33.25332593917847}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/18/2023 22:17:00", "git_sha": "fc4826f2", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 42.09002653757731, "generate_output_len_bytes": 2384, "generate_time": 20.106103817621868}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 8, "reps": 3, "date": "08/18/2023 22:20:31", "git_sha": "fc4826f2", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 185.28164370854697, "generate_output_len_bytes": 2772, "generate_time": 95.13023789723714}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 8, "reps": 3, "date": "08/18/2023 22:34:58", "git_sha": "fc4826f2", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 60.9919019540151, "generate_output_len_bytes": 2927, "generate_time": 34.328625202178955}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:31:34", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 52.49842747052511, "generate_output_len_bytes": 2172, "generate_time": 20.686774571736652}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:31:55", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:35:38", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1007, "summarize_time": 168.9666860898336, "generate_output_len_bytes": 2249, "generate_time": 73.25518870353699}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:48:09", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 856, "summarize_time": 45.30513469378153, "generate_output_len_bytes": 1802, "generate_time": 22.000216643015545}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 13:51:56", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 51.64275654157003, "generate_output_len_bytes": 2172, "generate_time": 20.737667481104534}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:35:47", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 980, "summarize_time": 280.4669913450877, "generate_output_len_bytes": 2132, "generate_time": 141.7793349424998}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 13:57:35", "git_sha": "fc4826f2", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 869, "summarize_time": 96.61887431144714, "generate_output_len_bytes": 3244, "generate_time": 82.98751719792683}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/18/2023 13:55:51", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1007, "summarize_time": 167.52292919158936, "generate_output_len_bytes": 2249, "generate_time": 71.82611886660258}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 14:08:08", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 856, "summarize_time": 47.14254776636759, "generate_output_len_bytes": 1802, "generate_time": 22.54850967725118}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 14:15:15", "git_sha": "d13230ee", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 14:07:15", "git_sha": "fc4826f2", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 89.59958203633626, "generate_output_len_bytes": 2172, "generate_time": 42.32424934705099}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 14:15:30", "git_sha": "d13230ee", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1024, "summarize_time": 185.44230167071024, "generate_output_len_bytes": 2122, "generate_time": 88.11553311347961}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 14:29:36", "git_sha": "d13230ee", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 922, "summarize_time": 68.06459252039592, "generate_output_len_bytes": 1802, "generate_time": 27.939613421758015}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/18/2023 14:26:29", "git_sha": "d13230ee", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 980, "summarize_time": 280.8310640652974, "generate_output_len_bytes": 2132, "generate_time": 143.21916349728903}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 14:48:17", "git_sha": "d13230ee", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 869, "summarize_time": 98.47045453389485, "generate_output_len_bytes": 3244, "generate_time": 83.71360301971436}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 15:35:13", "git_sha": "0dec0f52", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 15:49:33", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 16:26:53", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/18/2023 16:27:32", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 16:29:03", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 17:26:02", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 18:59:16", "git_sha": "5691db4a", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1075, "summarize_time": 39.01545596122742, "generate_output_len_bytes": 2242, "generate_time": 10.151424566904703}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 19:03:13", "git_sha": "5691db4a", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 940, "summarize_time": 21.78233750661214, "generate_output_len_bytes": 2130, "generate_time": 15.794983307520548}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 19:38:40", "git_sha": "6f05e8f1", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1114, "summarize_time": 7.636120955149333, "generate_output_len_bytes": 2275, "generate_time": 7.922623078028361}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/18/2023 19:41:02", "git_sha": "6f05e8f1", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1024, "summarize_time": 10.824170271555582, "generate_output_len_bytes": 2130, "generate_time": 9.209020694096884}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 19:55:17", "git_sha": "2c548f21", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA GeForce RTX 4090 (24564 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1088, "summarize_time": 24.39883820215861, "generate_output_len_bytes": 2275, "generate_time": 12.755743900934855}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/19/2023 00:57:21", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 37.113919814427696, "generate_output_len_bytes": 2384, "generate_time": 18.36507821083069}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:00:31", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 49.79721482594808, "generate_output_len_bytes": 2172, "generate_time": 21.780913591384888}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:04:36", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:05:26", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 181.2461258570353, "generate_output_len_bytes": 2772, "generate_time": 92.64811905225118}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:19:33", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 800, "summarize_time": 174.4576851526896, "generate_output_len_bytes": 2713, "generate_time": 119.14412077267964}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:36:14", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 53.39731526374817, "generate_output_len_bytes": 2927, "generate_time": 31.369641542434692}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:40:53", "git_sha": "a227be4f", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 74.27096923192342, "generate_output_len_bytes": 1802, "generate_time": 29.860486666361492}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 01:48:09", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 39.926851193110146, "generate_output_len_bytes": 2384, "generate_time": 18.481745958328247}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 01:51:27", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 51.299002488454185, "generate_output_len_bytes": 2172, "generate_time": 21.828503131866455}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/19/2023 01:56:20", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 178.19972308476767, "generate_output_len_bytes": 2772, "generate_time": 91.73426882425944}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/19/2023 02:10:13", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 800, "summarize_time": 180.7814578215281, "generate_output_len_bytes": 2713, "generate_time": 124.72717420260112}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/19/2023 02:26:43", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 57.08081785837809, "generate_output_len_bytes": 2927, "generate_time": 32.26534946759542}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/19/2023 02:31:36", "git_sha": "a227be4f", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 79.9461121559143, "generate_output_len_bytes": 1802, "generate_time": 31.403561115264893}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/19/2023 02:38:23", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 42.33977222442627, "generate_output_len_bytes": 2384, "generate_time": 19.723278522491455}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/19/2023 02:41:52", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 55.377869288126625, "generate_output_len_bytes": 2172, "generate_time": 25.01458676656087}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/19/2023 02:47:05", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 180.53432401021323, "generate_output_len_bytes": 2772, "generate_time": 91.93375285466512}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:01:07", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 800, "summarize_time": 179.50477250417075, "generate_output_len_bytes": 2713, "generate_time": 124.40728378295898}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:17:36", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 58.62867816289266, "generate_output_len_bytes": 2927, "generate_time": 33.394495725631714}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:22:37", "git_sha": "a227be4f", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 78.90612125396729, "generate_output_len_bytes": 1802, "generate_time": 30.697617371877033}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/19/2023 03:29:20", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 40.498607873916626, "generate_output_len_bytes": 2384, "generate_time": 19.509677171707153}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/19/2023 03:32:44", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 55.3964786529541, "generate_output_len_bytes": 2172, "generate_time": 24.347585439682007}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 8, "reps": 3, "date": "08/19/2023 03:37:55", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1179, "summarize_time": 186.71331850687662, "generate_output_len_bytes": 2772, "generate_time": 95.784650405248}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 8, "reps": 3, "date": "08/19/2023 03:52:28", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 800, "summarize_time": 185.3280005455017, "generate_output_len_bytes": 2713, "generate_time": 125.91738017400105}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 8, "reps": 3, "date": "08/19/2023 04:09:18", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1002, "summarize_time": 60.18280680974325, "generate_output_len_bytes": 2927, "generate_time": 33.386961142222084}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 8, "reps": 3, "date": "08/19/2023 04:14:25", "git_sha": "a227be4f", "n_gpus": 8, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 83.04790727297465, "generate_output_len_bytes": 1802, "generate_time": 32.24992283185323}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 23:26:19", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 47.03754989306132, "generate_output_len_bytes": 2384, "generate_time": 19.964784463246662}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 23:33:09", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 71.91136892636617, "generate_output_len_bytes": 2480, "generate_time": 33.6295014222463}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 23:44:08", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 00:45:42", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1007, "summarize_time": 148.61560583114624, "generate_output_len_bytes": 2357, "generate_time": 89.01266026496887}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 00:58:00", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 763, "summarize_time": 193.99270629882812, "generate_output_len_bytes": 2129, "generate_time": 95.66660761833191}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:13:01", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:13:55", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 991, "summarize_time": 61.52411222457886, "generate_output_len_bytes": 2927, "generate_time": 32.030215660730995}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/19/2023 01:19:00", "git_sha": "0cdb75ef", "n_gpus": 1, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 81.13888708750407, "generate_output_len_bytes": 3486, "generate_time": 55.5331826210022}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 01:27:49", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 47.41046245892843, "generate_output_len_bytes": 2384, "generate_time": 20.660600344340008}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 01:34:28", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 72.85646979014079, "generate_output_len_bytes": 2480, "generate_time": 34.05861854553223}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/19/2023 02:39:22", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1007, "summarize_time": 152.54357608159384, "generate_output_len_bytes": 2357, "generate_time": 91.51808977127075}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/19/2023 02:52:58", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 763, "summarize_time": 195.92926557858786, "generate_output_len_bytes": 2129, "generate_time": 96.55542047818501}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/19/2023 03:15:01", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 991, "summarize_time": 64.64422671000163, "generate_output_len_bytes": 2927, "generate_time": 33.30378039677938}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/19/2023 03:20:19", "git_sha": "0cdb75ef", "n_gpus": 2, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 84.57761120796204, "generate_output_len_bytes": 3486, "generate_time": 57.59072462717692}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:28:44", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1417, "summarize_time": 49.08898218472799, "generate_output_len_bytes": 2384, "generate_time": 21.489527861277264}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:32:39", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 74.43774898846944, "generate_output_len_bytes": 2480, "generate_time": 34.72673638661703}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:39:21", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1007, "summarize_time": 153.41076453526816, "generate_output_len_bytes": 2357, "generate_time": 91.14894040425618}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/19/2023 03:52:00", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 763, "summarize_time": 199.79869039853415, "generate_output_len_bytes": 2129, "generate_time": 98.61504419644673}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/19/2023 04:08:12", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 991, "summarize_time": 66.49260465304057, "generate_output_len_bytes": 2927, "generate_time": 34.17951035499573}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/19/2023 04:13:39", "git_sha": "0cdb75ef", "n_gpus": 4, "transformers": "4.30.2", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1000, "summarize_time": 87.65787092844646, "generate_output_len_bytes": 3486, "generate_time": 59.3750696182251}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/18/2023 22:22:24", "git_sha": "b63768c6", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "1 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 948, "summarize_time": 122.13213857014973, "generate_output_len_bytes": 2826, "generate_time": 66.34098903338115}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/18/2023 22:33:33", "git_sha": "c1348fb3", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "2 x NVIDIA RTX 6000 Ada Generation (49140 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 948, "summarize_time": 120.53812781969707, "generate_output_len_bytes": 2826, "generate_time": 67.28052496910095}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 22:56:52", "git_sha": "fb84de76", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1036, "summarize_time": 29.128981749216717, "generate_output_len_bytes": 2242, "generate_time": 12.197122732798258}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/18/2023 23:00:33", "git_sha": "fb84de76", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "1 x NVIDIA GeForce RTX 3090 (24576 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/19/2023 05:47:43", "git_sha": "22352acd", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 1, "reps": 3, "date": "08/19/2023 05:48:58", "git_sha": "22352acd", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 1, "reps": 3, "date": "08/19/2023 05:50:40", "git_sha": "22352acd", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 948, "summarize_time": 165.05752809842429, "generate_output_len_bytes": 2605, "generate_time": 93.80659619967143}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 06:05:51", "git_sha": "22352acd", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 2, "reps": 3, "date": "08/19/2023 06:10:05", "git_sha": "22352acd", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 906, "summarize_time": 410.0691332022349, "generate_output_len_bytes": 521, "generate_time": 57.71272214253744}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 2, "reps": 3, "date": "08/19/2023 06:36:58", "git_sha": "22352acd", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 948, "summarize_time": 171.74388321240744, "generate_output_len_bytes": 2605, "generate_time": 97.00725762049358}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/19/2023 06:51:13", "git_sha": "22352acd", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 792, "summarize_time": 267.0555826822917, "generate_output_len_bytes": 2783, "generate_time": 163.99818523724875}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 4, "reps": 3, "date": "08/19/2023 07:13:35", "git_sha": "22352acd", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 906, "summarize_time": 413.9569679101308, "generate_output_len_bytes": 521, "generate_time": 58.52583885192871}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 4, "reps": 3, "date": "08/19/2023 07:38:02", "git_sha": "22352acd", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 948, "summarize_time": 175.4907926718394, "generate_output_len_bytes": 2605, "generate_time": 98.97720170021057}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/19/2023 12:35:08", "git_sha": "29a002e5", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "timemachine", "gpus": "2 x NVIDIA GeForce RTX 3090 (24576 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 983, "summarize_time": 42.21107586224874, "generate_output_len_bytes": 2130, "generate_time": 16.94527777036031}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/21/2023 20:03:36", "git_sha": "51318f44", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 41.0461368560791, "generate_output_len_bytes": 2383, "generate_time": 19.614749511082966}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 20:07:35", "git_sha": "51318f44", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 42.8376894791921, "generate_output_len_bytes": 2383, "generate_time": 20.2719091574351}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/21/2023 20:42:46", "git_sha": "2f4bb620", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 20:50:19", "git_sha": "2f4bb620", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 66.52468911806743, "generate_output_len_bytes": 2479, "generate_time": 29.828714847564697}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 20:56:04", "git_sha": "2f4bb620", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "4 x NVIDIA RTX A6000 (46068 MiB)", "exception": "OOM"}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/21/2023 19:55:35", "git_sha": "51318f44", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 38.753786404927574, "generate_output_len_bytes": 2383, "generate_time": 19.529522736867268}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/21/2023 20:36:13", "git_sha": "51318f44", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 41.024452924728394, "generate_output_len_bytes": 2383, "generate_time": 20.29120985666911}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/21/2023 20:40:08", "git_sha": "51318f44", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 54.554532527923584, "generate_output_len_bytes": 2171, "generate_time": 24.604793945948284}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 20:50:05", "git_sha": "51318f44", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 41.09950613975525, "generate_output_len_bytes": 2383, "generate_time": 20.947362899780273}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 20:54:08", "git_sha": "51318f44", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 58.3172922929128, "generate_output_len_bytes": 2171, "generate_time": 25.735217014948528}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/21/2023 21:01:04", "git_sha": "51318f44", "n_gpus": 8, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 42.85940829912821, "generate_output_len_bytes": 2383, "generate_time": 21.380353291829426}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/21/2023 21:05:24", "git_sha": "51318f44", "n_gpus": 8, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 54.235164642333984, "generate_output_len_bytes": 2171, "generate_time": 25.70338026682536}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/21/2023 21:10:37", "git_sha": "51318f44", "n_gpus": 8, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 927, "summarize_time": 133.53030570348105, "generate_output_len_bytes": 2782, "generate_time": 72.97924383481343}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-70b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 22:18:17", "git_sha": "51318f44", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 927, "summarize_time": 131.45291074117026, "generate_output_len_bytes": 2782, "generate_time": 72.30849742889404}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/21/2023 22:51:09", "git_sha": "383b6bbc", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 39.269713958104454, "generate_output_len_bytes": 2383, "generate_time": 19.65731406211853}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/21/2023 22:54:54", "git_sha": "383b6bbc", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "1 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 51.84283971786499, "generate_output_len_bytes": 2171, "generate_time": 28.441521485646565}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/21/2023 23:13:10", "git_sha": "383b6bbc", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "2 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 53.383726040522255, "generate_output_len_bytes": 2171, "generate_time": 24.422890504201252}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 4, "reps": 3, "date": "08/21/2023 23:18:04", "git_sha": "383b6bbc", "n_gpus": 4, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "4 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 52.791220347086586, "generate_output_len_bytes": 2171, "generate_time": 25.378511508305866}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 8, "reps": 3, "date": "08/21/2023 23:23:11", "git_sha": "383b6bbc", "n_gpus": 8, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.8", "hostname": "cloudvm", "gpus": "8 x NVIDIA A100-SXM4-80GB (81920 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1046, "summarize_time": 56.3846542040507, "generate_output_len_bytes": 2171, "generate_time": 26.636192480723064}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 1, "reps": 3, "date": "08/21/2023 23:52:44", "git_sha": "da69b822", "n_gpus": 1, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "1 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1267, "summarize_time": 40.36223220825195, "generate_output_len_bytes": 2383, "generate_time": 19.87660264968872}
+{"backend": "text-generation-inference", "base_model": "h2oai/h2ogpt-4096-llama2-13b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 2, "reps": 3, "date": "08/22/2023 00:15:05", "git_sha": "e843e8c3", "n_gpus": 2, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "recypabaszmhhmuae", "gpus": "2 x NVIDIA RTX A6000 (46068 MiB)", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 915, "summarize_time": 64.78201874097188, "generate_output_len_bytes": 2479, "generate_time": 29.02147897084554}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 16, "ngpus": 0, "reps": 3, "date": "08/22/2023 19:01:15", "git_sha": "855b7d15", "n_gpus": 0, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "CPU", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1351, "summarize_time": 1215.5185990333557, "generate_output_len_bytes": 849, "generate_time": 180.56836318969727}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 8, "ngpus": 0, "reps": 3, "date": "08/22/2023 20:11:16", "git_sha": "855b7d15", "n_gpus": 0, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "CPU", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1353, "summarize_time": 1216.9783231417339, "generate_output_len_bytes": 849, "generate_time": 180.42225472132364}
+{"backend": "transformers", "base_model": "h2oai/h2ogpt-4096-llama2-7b-chat", "task": "summary_and_generate", "bits": 4, "ngpus": 0, "reps": 3, "date": "08/22/2023 21:21:20", "git_sha": "855b7d15", "n_gpus": 0, "transformers": "4.31.0", "bitsandbytes": "0.41.1", "cuda": "11.7", "hostname": "rippa", "gpus": "CPU", "summarize_input_len_bytes": 857252, "summarize_output_len_bytes": 1354, "summarize_time": 1217.1687794526417, "generate_output_len_bytes": 843, "generate_time": 180.78463260332742}

benchmarks/perf.md

@@ -0,0 +1,200 @@
+# Backend: transformers
+
+For [Interactive visualization of the results](https://raw.githubusercontent.com/h2oai/h2ogpt/blob/main/benchmarks/llm_gpu_benchmark_transformers.html), save the linked file as html on your machine and open it in a browser.
+
+
+## Model: h2oai/h2ogpt-4096-llama2-7b-chat (transformers)
+### Number of GPUs: 0
+|   bits | gpus   |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-------|---------------------------:|--------------------------------:|:------------|
+|     16 | CPU    |                    1215.52 |                         1.17546 |             |
+|      8 | CPU    |                    1216.98 |                         1.17641 |             |
+|      4 | CPU    |                    1217.17 |                         1.16575 |             |
+### Number of GPUs: 1
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    31.8619 |                        41.9433  |             |
+|     16 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                    32.2947 |                        40.9252  |             |
+|     16 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    37.1139 |                        32.4529  |             |
+|     16 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    47.0375 |                        29.8526  |             |
+|     16 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    67.9752 |                        18.0571  |             |
+|      8 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                   114.622  |                         9.21246 |             |
+|      8 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    94.1774 |                         8.95532 |             |
+|      8 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                   181.246  |                         7.47991 |             |
+|      8 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                   148.616  |                         6.61984 |             |
+|      8 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   185.146  |                         4.35807 |             |
+|      4 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                    39.544  |                        32.571   |             |
+|      4 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    42.8067 |                        32.3408  |             |
+|      4 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    53.3973 |                        23.3267  |             |
+|      4 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    61.5241 |                        22.8456  |             |
+|      4 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    90.5194 |                        14.9456  |             |
+### Number of GPUs: 2
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    32.1395 |                        40.3871  |             |
+|     16 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    39.9269 |                        32.248   |             |
+|     16 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    47.4105 |                        28.8472  |             |
+|     16 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    71.4808 |                        17.7518  |             |
+|      8 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    94.9813 |                         9.03765 |             |
+|      8 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                   178.2    |                         7.55443 |             |
+|      8 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                   152.544  |                         6.43862 |             |
+|      8 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   186.884  |                         4.35012 |             |
+|      4 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    43.235  |                        32.0566  |             |
+|      4 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    57.0808 |                        22.6791  |             |
+|      4 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    64.6442 |                        21.972   |             |
+|      4 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    94.5099 |                        14.6162  |             |
+### Number of GPUs: 4
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    42.3398 |                        30.2181  |             |
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    49.089  |                        27.7344  |             |
+|      8 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                   180.534  |                         7.53804 |             |
+|      8 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                   153.411  |                         6.46469 |             |
+|      4 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    58.6287 |                        21.9123  |             |
+|      4 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    66.4926 |                        21.409   |             |
+### Number of GPUs: 8
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    40.4986 |                        30.5489  |             |
+|      8 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                   186.713  |                         7.23498 |             |
+|      4 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    60.1828 |                        21.9172  |             |
+## Model: h2oai/h2ogpt-4096-llama2-13b-chat (transformers)
+### Number of GPUs: 1
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    52.4984 |                        26.2487  |             |
+|     16 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    49.7972 |                        24.9301  |             |
+|     16 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    71.9114 |                        18.4362  |             |
+|     16 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   nan      |                       nan       | OOM         |
+|     16 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                   nan      |                       nan       | OOM         |
+|      8 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                   168.967  |                         7.67522 |             |
+|      8 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                   185.442  |                         6.0205  |             |
+|      8 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                   174.458  |                         5.69269 |             |
+|      8 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                   193.993  |                         5.56359 |             |
+|      8 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   280.467  |                         3.75936 |             |
+|      4 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    45.3051 |                        20.4771  |             |
+|      4 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                    68.0646 |                        16.1241  |             |
+|      4 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    81.1389 |                        15.6933  |             |
+|      4 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    74.271  |                        15.0868  |             |
+|      4 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    96.6189 |                         9.77255 |             |
+### Number of GPUs: 2
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    51.6428 |                        26.1842  |             |
+|     16 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    51.299  |                        24.8757  |             |
+|     16 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    72.8565 |                        18.2039  |             |
+|     16 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    89.5996 |                        12.8295  |             |
+|      8 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                   167.523  |                         7.82793 |             |
+|      8 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                   195.929  |                         5.51238 |             |
+|      8 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                   180.781  |                         5.43787 |             |
+|      8 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   280.831  |                         3.72157 |             |
+|      4 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    47.1425 |                        19.9791  |             |
+|      4 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    84.5776 |                        15.1326  |             |
+|      4 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    79.9461 |                        14.3455  |             |
+|      4 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    98.4705 |                         9.68779 |             |
+### Number of GPUs: 4
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    55.3779 |                        21.7073  |             |
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    74.4377 |                        17.8537  |             |
+|      8 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                   179.505  |                         5.45185 |             |
+|      8 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                   199.799  |                         5.39725 |             |
+|      4 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    87.6579 |                        14.6779  |             |
+|      4 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    78.9061 |                        14.6754  |             |
+### Number of GPUs: 8
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    55.3965 |                        22.302   |             |
+|      8 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                   185.328  |                         5.38647 |             |
+|      4 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    83.0479 |                        13.969   |             |
+## Model: h2oai/h2ogpt-4096-llama2-70b-chat (transformers)
+### Number of GPUs: 1
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    nan     |                       nan       | OOM         |
+|     16 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    nan     |                       nan       | OOM         |
+|     16 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    nan     |                       nan       | OOM         |
+|     16 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    nan     |                       nan       | OOM         |
+|      8 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    nan     |                       nan       | OOM         |
+|      8 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    nan     |                       nan       | OOM         |
+|      8 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    nan     |                       nan       | OOM         |
+|      4 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    122.132 |                        10.6495  |             |
+|      4 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    165.058 |                         6.94248 |             |
+|      4 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    nan     |                       nan       | OOM         |
+### Number of GPUs: 2
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    nan     |                       nan       | OOM         |
+|      8 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    410.069 |                         2.25687 |             |
+|      4 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    120.538 |                        10.5008  |             |
+|      4 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    171.744 |                         6.71342 |             |
+|      4 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    nan     |                       nan       | OOM         |
+### Number of GPUs: 4
+|   bits | gpus                             |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:---------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB) |                    267.056 |                         4.24242 |             |
+|      8 | 4 x NVIDIA RTX A6000 (46068 MiB) |                    413.957 |                         2.22551 |             |
+|      4 | 4 x NVIDIA RTX A6000 (46068 MiB) |                    175.491 |                         6.5798  |             |
+# Backend: text-generation-inference
+
+For [Interactive visualization of the results](https://raw.githubusercontent.com/h2oai/h2ogpt/blob/main/benchmarks/llm_gpu_benchmark_text-generation-inference.html), save the linked file as html on your machine and open it in a browser.
+
+
+## Model: h2oai/h2ogpt-4096-llama2-7b-chat (text-generation-inference)
+### Number of GPUs: 1
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    39.0155 |                         55.2139 |             |
+|     16 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    29.129  |                         45.9535 |             |
+|     16 | 1 x NVIDIA GeForce RTX 4090 (24564 MiB)        |                    24.3988 |                         44.5878 |             |
+|     16 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    39.2697 |                         30.3068 |             |
+|     16 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                    40.3622 |                         29.9724 |             |
+### Number of GPUs: 2
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    7.63612 |                         71.7881 |             |
+|     16 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                   41.0461  |                         30.3726 |             |
+|     16 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                   41.0245  |                         29.36   |             |
+### Number of GPUs: 4
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    42.8377 |                         29.388  |             |
+|     16 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    41.0995 |                         28.4403 |             |
+### Number of GPUs: 8
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    42.8594 |                         27.8644 |             |
+## Model: h2oai/h2ogpt-4096-llama2-13b-chat (text-generation-inference)
+### Number of GPUs: 1
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 1 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    21.7823 |                         33.7132 |             |
+|     16 | 1 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    51.8428 |                         19.083  |             |
+|     16 | 1 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                   nan      |                        nan      | OOM         |
+|     16 | 1 x NVIDIA RTX A6000 (46068 MiB)               |                   nan      |                        nan      | OOM         |
+### Number of GPUs: 2
+|   bits | gpus                                           |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:-----------------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 2 x NVIDIA RTX 6000 Ada Generation (49140 MiB) |                    10.8242 |                         57.8237 |             |
+|     16 | 2 x NVIDIA GeForce RTX 3090 (24576 MiB)        |                    42.2111 |                         31.4247 |             |
+|     16 | 2 x NVIDIA A100-SXM4-80GB (81920 MiB)          |                    53.3837 |                         22.223  |             |
+|     16 | 2 x NVIDIA RTX A6000 (46068 MiB)               |                    64.782  |                         21.3549 |             |
+### Number of GPUs: 4
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    52.7912 |                         21.3862 |             |
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    66.5247 |                         20.777  |             |
+### Number of GPUs: 8
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    56.3847 |                         20.3764 |             |
+## Model: h2oai/h2ogpt-4096-llama2-70b-chat (text-generation-inference)
+### Number of GPUs: 4
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 4 x NVIDIA A100-SXM4-80GB (81920 MiB) |                    131.453 |                         9.61851 |             |
+|     16 | 4 x NVIDIA RTX A6000 (46068 MiB)      |                    nan     |                       nan       | OOM         |
+### Number of GPUs: 8
+|   bits | gpus                                  |   summarization time [sec] |   generation speed [tokens/sec] | exception   |
+|-------:|:--------------------------------------|---------------------------:|--------------------------------:|:------------|
+|     16 | 8 x NVIDIA A100-SXM4-80GB (81920 MiB) |                     133.53 |                         9.53011 |             |

blog/README.md

@@ -0,0 +1,81 @@
+# Building the World's Best Open-Source Large Language Model: H2O.ai's Journey
+
+by Arno Candel, PhD, CTO H2O.ai, April 19 2023
+
+At H2O.ai, we pride ourselves on developing world-class Machine Learning, Deep Learning, and AI platforms. We released H2O, the most widely used open-source distributed and scalable machine learning platform, before XGBoost, TensorFlow and PyTorch existed. H2O.ai is home to over 25 Kaggle grandmasters, including the current #1. In 2017, we used GPUs to create the world's best AutoML in H2O Driverless AI. We have witnessed first-hand how Large Language Models (LLMs) have taken over the world by storm.
+
+We are proud to announce that we are building h2oGPT, an LLM that not only excels in performance but is also fully open-source and commercially usable, providing a valuable resource for developers, researchers, and organizations worldwide.
+
+In this blog, we'll explore our journey in building h2oGPT in our effort to further democratize AI.
+
+## Why Open-Source LLMs?
+
+While LLMs like OpenAI's ChatGPT/GPT-4, Anthropic's Claude, Microsoft's Bing AI Chat, Google's Bard, and Cohere are powerful and effective, they have certain limitations compared to open-source LLMs:
+
+1. **Data Privacy and Security**: Using hosted LLMs requires sending data to external servers. This can raise concerns about data privacy, security, and compliance, especially for sensitive information or industries with strict regulations.
+2. **Dependency and Customization**: Hosted LLMs often limit the extent of customization and control, as users rely on the service provider's infrastructure and predefined models. Open-source LLMs allow users to tailor the models to their specific needs, deploy on their own infrastructure, and even modify the underlying code.
+3. **Cost and Scalability**: Hosted LLMs usually come with usage fees, which can increase significantly with large-scale applications. Open-source LLMs can be more cost-effective, as users can scale the models on their own infrastructure without incurring additional costs from the service provider.
+4. **Access and Availability**: Hosted LLMs may be subject to downtime or limited availability, affecting users' access to the models. Open-source LLMs can be deployed on-premises or on private clouds, ensuring uninterrupted access and reducing reliance on external providers.
+
+Overall, open-source LLMs offer greater flexibility, control, and cost-effectiveness, while addressing data privacy and security concerns. They foster a competitive landscape in the AI industry and empower users to innovate and customize models to suit their specific needs.
+
+## The H2O.ai LLM Ecosystem
+
+Our open-source LLM ecosystem currently includes the following components:
+
+1. **Code, data, and models**: Fully permissive, commercially usable [code](https://github.com/h2oai/h2ogpt), curated fine-tuning [data](https://huggingface.co/h2oai), and fine-tuned [models](https://huggingface.co/h2oai) ranging from 7 to 20 billion parameters.
+2. **State-of-the-art fine-tuning**: We provide code for highly efficient fine-tuning, including targeted data preparation, prompt engineering, and computational optimizations to fine-tune LLMs with up to 20 billion parameters (even larger models expected soon) in hours on commodity hardware or enterprise servers. Techniques like low-rank approximations (LoRA) and data compression allow computational savings of several orders of magnitude.
+3. **Chatbot**: We provide code to run a multi-tenant chatbot on GPU servers, with an easily shareable end-point and a Python client API, allowing you to evaluate and compare the performance of fine-tuned LLMs.
+4. **H2O LLM Studio**: Our no-code LLM fine-tuning framework created by the world's top Kaggle grandmasters makes it even easier to fine-tune and evaluate LLMs.
+
+Everything we release is based on fully permissive data and models, with all code open-sourced, enabling broader access for businesses and commercial products without legal concerns, thus expanding access to cutting-edge AI while adhering to licensing requirements.
+
+## Roadmap and Future Plans
+
+We have an ambitious roadmap for our LLM ecosystem, including:
+
+1. Integration with downstream applications and low/no-code platforms (H2O Document AI, H2O LLM Studio, etc.)
+2. Improved validation and benchmarking frameworks of LLMs
+3. Complementing our chatbot with search and other APIs (LangChain, etc.)
+4. Contribute to large-scale data cleaning efforts (Open Assistant, Stability AI, RedPajama, etc.)
+5. High-performance distributed training of larger models on trillion tokens
+6. High-performance scalable on-premises hosting for high-throughput endpoints
+7. Improvements in code completion, reasoning, mathematics, factual correctness, hallucinations, and reducing repetitions
+
+## Getting Started with H2O.ai's LLMs
+
+You can [Chat with h2oGPT](https://gpt.h2o.ai/) right now!
+
+https://user-images.githubusercontent.com/6147661/232924684-6c0e2dfb-2f24-4098-848a-c3e4396f29f6.mov
+
+![](https://user-images.githubusercontent.com/6147661/233239878-de3b0fce-5425-4189-8095-5313c7817d58.png)
+![](https://user-images.githubusercontent.com/6147661/233239861-e99f238c-dd5d-4dd7-ac17-6367f91f86ac.png)
+
+To start using our LLM as a developer, follow the steps below:
+
+1. Clone the repository: `git clone https://github.com/h2oai/h2ogpt.git`
+2. Change to the repository directory: `cd h2ogpt`
+3. Install the requirements: `pip install -r requirements.txt`
+4. Run the chatbot: `python generate.py --base_model=h2oai/h2ogpt-oig-oasst1-256-6_9b`
+5. Open your browser at `http://0.0.0.0:7860` or the public live URL printed by the server.
+
+For more information, visit [h2oGPT GitHub page](https://github.com/h2oai/h2ogpt), [H2O.ai's Hugging Face page](https://huggingface.co/h2oai) and [H2O LLM Studio GitHub page](https://github.com/h2oai/h2o-llmstudio).
+
+Join us on this exciting journey as we continue to improve and expand the capabilities of our open-source LLM ecosystem!
+
+## Acknowledgements
+
+We appreciate the work by many open-source contributors, especially:
+
+* [H2O.ai makers](https://h2o.ai/company/team/)
+* [Alpaca-LoRA](https://github.com/tloen/alpaca-lora/)
+* [LoRA](https://github.com/microsoft/LoRA/)
+* [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca/)
+* [Hugging Face](https://huggingface.co/)
+* [OpenAssistant](https://open-assistant.io/)
+* [EleutherAI](https://www.eleuther.ai/)
+* [LAION](https://laion.ai/blog/oig-dataset/)
+* [BigScience](https://github.com/bigscience-workshop/bigscience/)
+* [LLaMa](https://github.com/facebookresearch/llama/)
+* [StableLM](https://github.com/Stability-AI/StableLM/)
+* [Vicuna](https://github.com/lm-sys/FastChat/)

ci/jenkinsfile

@@ -0,0 +1,158 @@
+#!/usr/bin/groovy
+
+@Library('test-shared-library@dai_pipeline') _
+
+import ai.h2o.ci.buildsummary.StagesSummary
+import groovy.json.JsonOutput
+
+buildSummary('https://github.com/h2oai/h2ogpt', true)
+buildSummary.get().addStagesSummary(this, new StagesSummary())
+
+def ALL_TESTS = [
+        "test_osx": [
+            install_deps: "TRAINING",
+            test_target: "test_imports",
+            node: "osx",
+            test_markers: "not need_tokens and not need_gpu",
+            timeout: 90,
+            use_docker: false,
+            env: ['PYTHON_BINARY=/Users/jenkins/anaconda/envs/h2ogpt-py3.10/bin/python']
+        ],
+        "test_all": [
+            install_deps: "TRAINING,WIKI_EXTRA",
+            test_target: "test",
+            test_markers: "not need_tokens and not need_gpu",
+            node: "DAIDEV-GPU || DAIDEV-2GPU",
+            timeout: 90,
+            use_docker: true,
+            env: []
+        ],
+]
+
+pipeline {
+    agent none
+    parameters {
+        booleanParam(name: 'skipTesting', defaultValue: false, description: 'Skip testing')
+        text(name: "testTargets", defaultValue: "${ALL_TESTS.keySet().join('\n')}", description: "A select set of tests to run")
+        booleanParam(name: 'publish', defaultValue: false, description: 'Upload to HF')
+    }
+    options {
+        ansiColor('xterm')
+        timestamps()
+    }
+    stages {
+        stage('Build') {
+            agent {
+                label "linux && docker"
+            }
+            steps {
+                script {
+                    def shortHash = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
+                    def commitMsg = sh(returnStdout: true, script: 'git log -1 --pretty=format:"[%an] %s"').trim()
+                    currentBuild.displayName = "${env.BUILD_ID} - [${shortHash}]"
+                    currentBuild.description = "${commitMsg}"
+
+                    sh "make docker_build"
+                    docker.image("harbor.h2o.ai/library/python:3.10").inside("--entrypoint='' --security-opt seccomp=unconfined -e USE_WHEEL=1 -e HOME=${WORKSPACE}") {
+                        sh "make clean dist"
+                    }
+
+                    archiveArtifacts allowEmptyArchive: true, artifacts: "dist/h2ogpt-*.whl"
+                    stash includes: "dist/h2ogpt-*.whl", name: "wheel_file"
+                }
+            }
+        }
+
+        stage('Tests') {
+            when {
+                anyOf {
+                    expression { return !params.skipTesting }
+                }
+                beforeAgent true
+            }
+            agent {
+                label "linux && docker"
+            }
+            steps {
+                script {
+                    def testTargets = [:]
+                    params.testTargets.split('\n').findAll{ it.contains("test_") }.each { testName ->
+                        testTargets[testName] = {
+                            node("${ALL_TESTS[testName].node}") {
+                                buildSummary.stageWithSummary("${testName}", "${testName}") {
+                                    buildSummary.setStageUrl("${testName}")
+                                    timeout(time: ALL_TESTS[testName].timeout, unit: 'MINUTES') {
+                                        script {
+                                            try {
+                                                dir("${testName}") {
+                                                    withEnv(ALL_TESTS[testName].env + ["PYTEST_TEST_NAME=_${testName}", "IS_PR_BUILD=${isPrBranch()}", "USE_WHEEL=1"]) {
+
+                                                        // cleanup and force the use of the installed wheel
+                                                        deleteDir()
+                                                        checkout scm
+                                                        unstash "wheel_file"
+                                                        sh "rm -rf *.py spaces models"
+
+                                                        // pull runtime details
+                                                        def dockerImage = sh(returnStdout: true, script: "make print-DOCKER_TEST_IMAGE").trim()
+                                                        def nvidiaSmiExitCode = sh(returnStdout: false, returnStatus: true, script: "nvidia-smi")
+                                                        // def dockerRuntime = "${nvidiaSmiExitCode}" == "0" ? "--runtime nvidia" : ""
+                                                        def dockerRuntime = ""  // TODO: keep until lab machines are upgraded
+
+                                                        if (ALL_TESTS[testName].use_docker) {
+                                                            docker.image("${dockerImage}").inside("--entrypoint='' --security-opt seccomp=unconfined --ulimit core=-1 --init --pid=host -e USE_WHEEL=1 -e HOME=${WORKSPACE}/${testName} ${dockerRuntime}") {
+                                                                sh "nvidia-smi || true"
+                                                                sh "SKIP_MANUAL_TESTS=1 PYTHON_BINARY=/usr/bin/python3.10 make install"
+                                                                sh "SKIP_MANUAL_TESTS=1 PYTHON_BINARY=/usr/bin/python3.10 make install-${ALL_TESTS[testName].install_deps}"
+                                                                sh """DEFAULT_MARKERS="${ALL_TESTS[testName].test_markers}" SKIP_MANUAL_TESTS=1 PYTHON_BINARY=/usr/bin/python3.10 make ${ALL_TESTS[testName].test_target}"""
+                                                            }
+                                                        } else {
+                                                            sh "make venv"
+                                                            sh "SKIP_MANUAL_TESTS=1 PYTHON_BINARY=${WORKSPACE}/${testName}/venv/bin/python make install"
+                                                            sh "SKIP_MANUAL_TESTS=1 PYTHON_BINARY=${WORKSPACE}/${testName}/venv/bin/python make install-${ALL_TESTS[testName].install_deps}"
+                                                            sh """DEFAULT_MARKERS="${ALL_TESTS[testName].test_markers}" SKIP_MANUAL_TESTS=1 PYTHON_BINARY=${WORKSPACE}/${testName}/venv/bin/python make ${ALL_TESTS[testName].test_target}"""
+                                                        }
+                                                    }
+                                                }
+                                            } catch (e) {
+                                                throw e
+                                            } finally {
+                                                sh "mv ${testName}/test_report.xml ${testName}/${testName}_report.xml"
+                                                archiveArtifacts allowEmptyArchive: true, artifacts: "${testName}/${testName}_report.xml"
+                                                junit testResults: "${testName}/${testName}_report.xml", keepLongStdio: true, allowEmptyResults: true
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+
+                    parallel(testTargets)
+                }
+            }
+        }
+
+        stage('Publish') {
+            when {
+                anyOf {
+                    expression { return params.publish }
+                }
+                beforeAgent true
+            }
+            agent {
+                label "linux && docker"
+            }
+            steps {
+                script {
+                    sh "make IS_PR_BUILD=${isPrBranch()} BUILD_NUMBER=${env.BUILD_ID} BUILD_BASE_NAME=${env.JOB_BASE_NAME} publish"
+                }
+            }
+        }
+    }
+}
+
+def isPrBranch() {
+    return (env.CHANGE_BRANCH != null && env.CHANGE_BRANCH != '') ||
+            (env.BRANCH_NAME != null && env.BRANCH_NAME.startsWith("PR-"))
+}

cloud/packer/Jenkinsfile

@@ -0,0 +1,80 @@
+import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
+
+properties(
+    [
+        parameters(
+            [
+                string(name: 'BRANCH_TAG', defaultValue: 'origin/main'),
+                booleanParam(name: 'AZURE', defaultValue: true, description: 'Make Azure Machine Image/Not?'),
+                booleanParam(name: 'GCP', defaultValue: true, description: 'Make GCP Image/Not?'),
+                string(name: 'H2OGPT_VERSION', defaultValue: "010", description: 'Example: for version 1.10.5 use 1105')
+            ]
+        )
+    ]
+)
+
+node('linux && docker') {
+    stage('Init') {
+        cleanWs()
+        currentBuild.displayName = "#${BUILD_NUMBER} - Rel:${H2OGPT_VERSION}"
+        checkout scm
+        sh('ls -al')
+    }
+
+    stage('Build Images') {
+        try {
+            docker.image('harbor.h2o.ai/opsh2oai/h2oai-packer-build:2').inside {
+                parallel([
+                        "GCP Ubuntu 20.04": {
+                            withCredentials([file(credentialsId: 'GCP_MARKETPLACE_SERVICE_ACCOUNT', variable: 'GCP_ACCOUNT_FILE')]) {
+                                dir('cloud/packer') {
+                                    if (params.GCP) {
+                                        sh("packer build \
+                                            --force \
+                                            -var 'project_id=h2o-gce' \
+                                            -var 'account_file=$GCP_ACCOUNT_FILE' \
+                                            -var 'h2ogpt_version=${H2OGPT_VERSION}' \
+                                            -var 'branch_tag=${BRANCH_TAG}' \
+                                            h2ogpt-gcp.json"
+                                        )
+                                        archiveArtifacts artifacts: '*-image-info.json'
+                                    }else {
+                                        Utils.markStageSkippedForConditional('GCP Ubuntu 20.04')
+                                    }
+                                }
+                            }
+                        },
+
+                         "AZURE Ubuntu 20.04": {
+                            withCredentials([string(credentialsId: "AZURE_MARKETPLACE_CLIENT_ID", variable: "AZURE_CLIENT_ID"),
+                                             string(credentialsId: "AZURE_MARKETPLACE_CLIENT_SECRET", variable: "AZURE_CLIENT_SECRET"),
+                                             string(credentialsId: "AZURE_MARKETPLACE_SUBSCRIPTION_ID", variable: "AZURE_SUBSCRIPTION_ID"),
+                                             string(credentialsId: "AZURE_MARKETPLACE_TENANT_ID", variable: "AZURE_TENANT_ID")]) {
+                                dir('cloud/packer') {
+                                    if (params.AZURE) {
+                                        sh("packer build \
+                                            --force \
+                                            -var 'client_id=$AZURE_CLIENT_ID' \
+                                            -var 'client_secret=$AZURE_CLIENT_SECRET' \
+                                            -var 'managed_image_resource_group_name=H2OIMAGES' \
+                                            -var 'subscription_id=$AZURE_SUBSCRIPTION_ID' \
+                                            -var 'tenant_id=$AZURE_TENANT_ID' \
+                                            -var 'h2ogpt_version=${H2OGPT_VERSION}' \
+                                            -var 'branch_tag=${BRANCH_TAG}' \
+                                            h2ogpt-azure.json"
+                                        )
+                                        archiveArtifacts artifacts: '*-image-info.json'
+                                    }else {
+                                        Utils.markStageSkippedForConditional('AZURE Ubuntu 20.04')
+                                    }
+                                }
+                            }
+                        },
+
+                ])
+            }
+        } finally {
+            cleanWs()
+        }
+    }
+}

cloud/packer/README.md

@@ -0,0 +1,22 @@
+# h2oGPT Packer Templates
+
+These scripts help create images in public clouds that can then submitted to Azure/GCP Marketplace for commercial use.
+
+### Packer Scripts 
+- Azure - `h2ogpt-azure.json`
+- GCP - `h2ogpt-gcp.json`
+
+### Provisioning Scripts
+ - `setup_environment.sh`
+    - Responsible for setting up CUDA, GCC, Nginx, Python
+- `install_h2ogpt.sh`
+    - Responsible for setting up h2oGPT with its dependencies
+- `h2oai-h2ogpt-4096-llama2-13b-chat.sh`
+    - Responsible for setting up default model h2oai-h2ogpt-4096-llama2-13b-chat with vLLM in port 80 via Nginx
+    - vLLM, h2oGPT and Nginx are executed through services
+    - Model is downloaded at the runtime
+
+__Jenkins Pipeline__: http://jenkins.h2o.local:8080/job/build-h2ogpt-cloud-images/
+
+### Notes:
+ - Since model is downloaded at the runtime after VM is provisioned it takes around 5 - 10 min start h2oGPT correctly

cloud/packer/h2oai-h2ogpt-4096-llama2-13b-chat.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+sudo systemctl daemon-reload
+sudo systemctl enable h2ogpt_nginx.service
+sudo systemctl enable vllm.service
+sudo systemctl enable h2ogpt.service
+
+cd "$HOME"
+# sudo rm -rf "$HOME"/.cache/huggingface/hub/
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y autoremove
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y clean

cloud/packer/h2ogpt-azure.json

@@ -0,0 +1,123 @@
+{
+    "variables": {
+        "client_id": "<AZURE CLIENT ID>",
+        "client_secret": "<AZURE CLIENT SECRET>",
+        "subscription_id": "92429150-401a-431f-8955-e69c0c119e68",
+        "tenant_id": "840229f2-c911-49e6-a73d-5b3a4311835a",
+        "managed_image_resource_group_name": "H2OIMAGES",
+        "h2ogpt_version": "010",
+        "branch_tag": "main",
+        "base_model": "h2oai-h2ogpt-4096-llama2-13b-chat"
+    },
+    "builders": [
+        {
+            "type": "azure-arm",
+            "client_id": "{{user `client_id`}}",
+            "client_secret": "{{user `client_secret`}}",
+            "subscription_id": "{{user `subscription_id`}}",
+            "tenant_id": "{{user `tenant_id`}}",
+            "capture_container_name": "h2ovhdimages",
+            "capture_name_prefix": "h2ogpt-{{user `h2ogpt_version`}}",
+            "resource_group_name": "{{user `managed_image_resource_group_name`}}",
+            "temp_resource_group_name": "Engineering_DevOps_h2oGPT-Ubuntu",
+            "storage_account": "h2ovhdimages",
+            "os_type": "Linux",
+            "image_publisher": "Canonical",
+            "image_offer": "0001-com-ubuntu-server-focal",
+            "image_sku": "20_04-lts",
+            "os_disk_size_gb": 512,
+            "azure_tags": {
+                "dept": "Engineering",
+                "task": "Image deployment",
+                "Name": "H2OGPT-CLOUD-IMAGES",
+                "Owner": "[email protected]",
+                "Project": "DevOps",
+                "Department": "Engineering",
+                "Environment": "Dev",
+                "Scheduling": "self-managed"
+            },
+            "location": "East US",
+            "vm_size": "Standard_NC24s_v3",
+            "ssh_username": "ubuntu"
+        }
+    ],
+    "post-processors": [
+        {
+            "type": "manifest",
+            "output": "azure-ubuntu-image-info.json",
+            "strip_path": true,
+            "custom_data": {
+                "base_image": "AZURE Ubuntu 20.04",
+                "h2ogpt_version": "{{user `h2ogpt_version`}}"
+            }
+        }
+    ],
+    "provisioners": [
+      {
+        "type": "shell",
+        "script": "setup_environment.sh",
+        "pause_before": "10s",
+        "pause_after": "10s"
+      },
+      {
+        "type": "shell",
+        "inline": ["sudo reboot now"],
+        "pause_after": "10s",
+        "expect_disconnect": true
+      },
+      {
+        "type": "shell",
+        "environment_vars": ["BRANCH_TAG={{user `branch_tag`}}"],
+        "script": "install_h2ogpt.sh",
+        "pause_after": "10s"
+      },
+      {
+        "type": "shell",
+        "inline": [
+          "sudo chown -R ubuntu:ubuntu /etc/nginx/conf.d",
+          "sudo chown -R ubuntu:ubuntu /etc/systemd/system/"
+        ],
+        "pause_before": "10s"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/run_nginx.sh",
+        "destination": "/workspace/run_nginx.sh"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/run_vllm.sh",
+        "destination": "/workspace/run_vllm.sh"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/run_h2ogpt.sh",
+        "destination": "/workspace/run_h2ogpt.sh"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/h2ogpt_nginx.service",
+        "destination": "/etc/systemd/system/h2ogpt_nginx.service"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/vllm.service",
+        "destination": "/etc/systemd/system/vllm.service"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/h2ogpt.service",
+        "destination": "/etc/systemd/system/h2ogpt.service"
+      },
+      {
+        "type":  "file",
+        "source": "./startup-scripts/temp.conf",
+        "destination": "/workspace/temp.conf"
+      },
+      {
+        "type": "shell",
+        "script": "{{user `base_model`}}.sh",
+        "pause_after": "10s"
+      }
+      ]
+}

cloud/packer/h2ogpt-gcp.json

@@ -0,0 +1,107 @@
+{
+  "variables": {
+    "project_id": "eng-llm",
+    "account_file": "<NAME OF GCP CREDENTIALS JSON FILE>",
+    "h2ogpt_version": "010",
+    "branch_tag": "main",
+    "base_model": "h2oai-h2ogpt-4096-llama2-13b-chat"
+  },
+  "builders": [
+    {
+      "type": "googlecompute",
+      "project_id": "{{user `project_id`}}",
+      "account_file": "{{user `account_file`}}",
+      "machine_type": "n1-standard-8",
+      "on_host_maintenance": "TERMINATE",
+      "accelerator_type": "projects/{{user `project_id`}}/zones/us-west1-b/acceleratorTypes/nvidia-tesla-t4",
+      "accelerator_count": "4",
+      "source_image_family": "ubuntu-2004-lts",
+      "zone": "us-west1-b",
+      "image_description": "h2ogpt using Packer",
+      "image_name": "h2ogpt-{{user `h2ogpt_version`}}",
+      "disk_size": 512,
+      "disk_type": "pd-ssd",
+      "ssh_username": "ubuntu",
+      "tags": ["h2ogpt"]
+    }
+  ],
+  "post-processors": [
+    {
+      "type": "manifest",
+      "output": "gcp-image-info.json",
+      "strip_path": true,
+      "custom_data": {
+        "base_image": "GCP Ubuntu 20.04",
+        "h2ogpt_version": "{{user `h2ogpt_version`}}"
+      }
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "script": "setup_environment.sh",
+      "pause_before": "10s",
+      "pause_after": "10s"
+    },
+    {
+      "type": "shell",
+      "inline": ["sudo reboot now"],
+      "pause_after": "10s",
+      "expect_disconnect": true
+    },
+    {
+      "type": "shell",
+      "environment_vars": ["BRANCH_TAG={{user `branch_tag`}}"],
+      "script": "install_h2ogpt.sh",
+      "pause_after": "10s"
+    },
+    {
+      "type": "shell",
+      "inline": [
+        "sudo chown -R ubuntu:ubuntu /etc/nginx/conf.d",
+        "sudo chown -R ubuntu:ubuntu /etc/systemd/system/"
+      ],
+      "pause_before": "10s"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/run_nginx.sh",
+      "destination": "/workspace/run_nginx.sh"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/run_vllm.sh",
+      "destination": "/workspace/run_vllm.sh"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/run_h2ogpt.sh",
+      "destination": "/workspace/run_h2ogpt.sh"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/h2ogpt_nginx.service",
+      "destination": "/etc/systemd/system/h2ogpt_nginx.service"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/vllm.service",
+      "destination": "/etc/systemd/system/vllm.service"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/h2ogpt.service",
+      "destination": "/etc/systemd/system/h2ogpt.service"
+    },
+    {
+      "type":  "file",
+      "source": "./startup-scripts/temp.conf",
+      "destination": "/workspace/temp.conf"
+    },
+    {
+      "type": "shell",
+      "script": "{{user `base_model`}}.sh",
+      "pause_after": "10s"
+    }
+  ]
+}

cloud/packer/install_h2ogpt.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+export PATH=$PATH:/home/ubuntu/.local/bin
+sudo mkdir -p /workspace && cd /workspace
+sudo chmod a+rwx .
+
+git config --global --add safe.directory /workspace
+git config --global advice.detachedHead false
+git clone https://github.com/h2oai/h2ogpt.git .
+
+if [ -z "$BRANCH_TAG" ]; then
+  echo "BRANCH_TAG environment variable is not set."
+  exit 1
+fi
+
+git checkout $BRANCH_TAG
+
+ls -la
+sudo ./docker_build_script_ubuntu.sh

cloud/packer/setup_environment.sh

@@ -0,0 +1,46 @@
+#!/bin/bash -e
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y update
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
+  git \
+  software-properties-common \
+  pandoc \
+  curl \
+  apt-utils \
+  make \
+  build-essential \
+  wget \
+  gnupg2 \
+  ca-certificates \
+  lsb-release \
+  ubuntu-keyring
+
+curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
+gpg --dry-run --quiet --no-keyring --import --import-options import-show /usr/share/keyrings/nginx-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" sudo tee /etc/apt/sources.list.d/nginx.list
+echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" sudo tee /etc/apt/preferences.d/99nginx
+
+sudo DEBIAN_FRONTEND=noninteractive apt -y update
+sudo DEBIAN_FRONTEND=noninteractive apt -y install nginx
+
+MAX_GCC_VERSION=11
+sudo DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:ubuntu-toolchain-r/test
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y install gcc-$MAX_GCC_VERSION g++-$MAX_GCC_VERSION
+
+sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$MAX_GCC_VERSION 100
+sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$MAX_GCC_VERSION 100
+sudo update-alternatives --set gcc /usr/bin/gcc-$MAX_GCC_VERSION
+sudo update-alternatives --set g++ /usr/bin/g++-$MAX_GCC_VERSION
+
+wget --quiet https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
+sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600
+wget --quiet https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda-repo-ubuntu2004-11-8-local_11.8.0-520.61.05-1_amd64.deb
+sudo dpkg -i cuda-repo-ubuntu2004-11-8-local_11.8.0-520.61.05-1_amd64.deb
+sudo cp /var/cuda-repo-ubuntu2004-11-8-local/cuda-*-keyring.gpg /usr/share/keyrings/
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y update
+sudo DEBIAN_FRONTEND=noninteractive apt-get -y install cuda
+sudo rm -rf "*.deb"
+
+sudo echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11.8/lib64/" >> ~/.bashrc
+sudo echo "export CUDA_HOME=/usr/local/cuda-11.8" >> ~/.bashrc
+sudo echo "export PATH=$PATH:/h2ogpt_conda/bin:/usr/local/cuda-11.8/bin/" >> ~/.bashrc

cloud/packer/startup-scripts/h2ogpt.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=h2oGPT Server
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/workspace
+ExecStart=/usr/bin/bash /workspace/run_h2ogpt.sh
+
+[Install]
+WantedBy=multi-user.target

cloud/packer/startup-scripts/h2ogpt_nginx.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=h2oGPT Nginx Server
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/workspace
+ExecStart=/usr/bin/bash /workspace/run_nginx.sh
+
+[Install]
+WantedBy=multi-user.target

cloud/packer/startup-scripts/run_h2ogpt.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+while true; do
+  http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/v1/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+      "model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+      "prompt": "San Francisco is a",
+      "max_tokens": 7,
+      "temperature": 0
+    }')
+
+  if [ "$http_code" -eq 200 ]; then
+    echo "Received HTTP 200 status code. Starting h2ogpt service"
+    CUDA_VISIBLE_DEVICES=$(seq -s, $(($(nvidia-smi -L | wc -l) / 2)) $(($(nvidia-smi -L | wc -l) - 1))) /h2ogpt_conda/bin/python3.10 \
+      /workspace/generate.py \
+      --inference_server="vllm:0.0.0.0:5000" \
+      --base_model=h2oai/h2ogpt-4096-llama2-13b-chat \
+      --langchain_mode=UserData
+    break
+  else
+    echo "Received HTTP $http_code status code. Retrying in 5 seconds..."
+    sleep 5
+  fi
+done
+

cloud/packer/startup-scripts/run_nginx.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+while true; do
+  http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:5000/v1/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+      "model": "h2oai/h2ogpt-4096-llama2-13b-chat",
+      "prompt": "San Francisco is a",
+      "max_tokens": 7,
+      "temperature": 0
+    }')
+
+  if [ "$http_code" -eq 200 ]; then
+    echo "Received HTTP 200 status code. Restarting Nginx for h2oGPT"
+    ip=$(dig +short myip.opendns.com @resolver1.opendns.com)
+    sed "s/<|_SUBST_PUBLIC_IP|>;/$ip;/g" /workspace/temp.conf  > /etc/nginx/conf.d/h2ogpt.conf
+    sudo systemctl restart nginx.service
+    break
+  else
+    echo "Received HTTP $http_code status code. Retrying in 5 seconds..."
+    sleep 5
+  fi
+done

cloud/packer/startup-scripts/run_vllm.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+tps=$(nvidia-smi --query-gpu=gpu_name --format=csv,noheader | wc -l | awk '{if ($1 > 1) print int($1/2); else print 1}')
+NCCL_IGNORE_DISABLED_P2P=1 CUDA_VISIBLE_DEVICES=$(seq -s, 0 $(($(nvidia-smi -L | wc -l) > 1 ? $(nvidia-smi -L | wc -l) / 2 - 1 : 0))) \
+/h2ogpt_conda/vllm_env/bin/python3.10 -m vllm.entrypoints.openai.api_server \
+    --port=5000 \
+    --host=0.0.0.0 \
+    --model h2oai/h2ogpt-4096-llama2-13b-chat \
+    --tokenizer=hf-internal-testing/llama-tokenizer \
+    --tensor-parallel-size=$tps --seed 1234

cloud/packer/startup-scripts/temp.conf

@@ -0,0 +1,14 @@
+server {
+    listen 80;
+    listen [::]:80;
+    server_name <|_SUBST_PUBLIC_IP|>;  # Change this to your domain name
+
+    location / {  # Change this if you'd like to server your Gradio app on a different path
+        proxy_pass http://0.0.0.0:7860/; # Change this if your Gradio app will be running on a different port
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+    }
+}

cloud/packer/startup-scripts/vllm.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=vLLM Server
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/workspace
+ExecStart=/usr/bin/bash /workspace/run_vllm.sh
+
+[Install]
+WantedBy=multi-user.target

data/NGSL_1.2_stats.csv.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34993377b20347d5c8837a1101bde4e403232f5f08c80f9441e16ac7a23228a7
+size 25168

data/README-template.md

@@ -0,0 +1,23 @@
+---
+license: apache-2.0
+language:
+- en
+thumbnail: https://h2o.ai/etc.clientlibs/h2o/clientlibs/clientlib-site/resources/images/favicon.ico
+tags:
+- gpt
+- llm
+- large language model
+- open-source
+---
+# h2oGPT Data Card
+## Summary
+
+H2O.ai's `<<DATASET_NAME>>` is an open-source instruct-type dataset for fine-tuning of large language models, licensed for commercial use.
+
+- Number of rows: `<<NROWS>>`
+- Number of columns: `<<NCOLS>>`
+- Column names: `<<COLNAMES>>`
+
+## Source
+
+<<SOURCE_LINK>>

data/censor_words.txt

@@ -0,0 +1,10 @@
+fuck-buddy
+fuck-buddys
+clusterfuck
+fuckup
+fuckups
+dumbfuck
+dumbfucks
+mindfuck
+*fucking
+fuckin'

data/count_1w.txt.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:434d0f21e4156386a3e9156c669e316878fbb1664414ffe06f22e254a180d363
+size 2234946

data/create_data_cards.py

@@ -0,0 +1,151 @@
+import shutil
+
+import pandas as pd
+import os
+
+import huggingface_hub
+import pytest
+from datasets import load_dataset
+
+
+@pytest.mark.parametrize(
+    "dataset_name, link_to_source",
+    [
+        (
+                "h2ogpt-oig-instruct-cleaned",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/b8f15efcc305a953c52a0ee25b8b4897ceb68c0a/scrape_dai_docs.py)
+"""
+        ),
+        (
+                "h2ogpt-oig-instruct-cleaned-v2",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/40c217f610766715acec297a5535eb440ac2f2e2/create_data.py)
+"""
+        ),
+        (
+                "h2ogpt-oig-instruct-cleaned-v3",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/bfc3778c8db938761ce2093351bf2bf82159291e/create_data.py)
+"""
+        ),
+        (
+                "openassistant_oasst1",
+                """
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/45e6183171fb16691ad7d3ab006fad973f971e98/create_data.py#L1253)
+"""
+        ),
+        (
+                "h2ogpt-oig-oasst1-instruct-cleaned-v1",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/main/docs/FINETUNE.md#high-quality-oig-based-instruct-data)
+
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/5fc91911bc2bfaaf3b6c2de577c4b0ae45a07a4a/create_data.py#L1253)
+"""
+        ),
+        (
+                "h2ogpt-oig-oasst1-instruct-cleaned-v2",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/main/docs/FINETUNE.md#high-quality-oig-based-instruct-data)
+
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/0e70c2fbb16410bd8e6992d879b4c55cd981211f/create_data.py#L1375-L1415)
+"""
+        ),
+        (
+                "h2ogpt-oig-oasst1-instruct-cleaned-v3",
+                """
+- [Original LAION OIG Dataset](https://github.com/LAION-AI/Open-Instruction-Generalist)
+- [LAION OIG data detoxed and filtered down by scripts in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/main/docs/FINETUNE.md#high-quality-oig-based-instruct-data)
+
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/6728938a262d3eb5e8db1f252bbcd7de838da452/create_data.py#L1415)
+"""
+        ),
+        (
+                "openassistant_oasst1_h2ogpt",
+                """
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/83857fcf7d3b712aad5db32207e6db0ab0f780f9/create_data.py#L1252)
+"""
+        ),
+        (
+                "openassistant_oasst1_h2ogpt_graded",
+                """
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/d1f8ce975a46056d41135d126dd33de8499aa26e/create_data.py#L1259)
+"""
+        ),
+        (
+                "h2ogpt-fortune2000-personalized",
+                """
+- [Fortune 2000 companies from Wikipedia](https://github.com/h2oai/h2ogpt/blob/b1ea74c0088884ebff97f1ccddbfb3f393e29e44/create_data.py#L1743)
+"""
+        ),
+        (
+                "openassistant_oasst1_h2ogpt_llama2_chat",
+                """
+- [Original Open Assistant data in tree structure](https://huggingface.co/datasets/OpenAssistant/oasst1)
+- [This flattened dataset created by script in h2oGPT repository](https://github.com/h2oai/h2ogpt/blob/0bee5f50a74f489ca3fc81486f9322078360f2cb/src/create_data.py#L1296)
+"""
+        ),
+    ],
+)
+def test_create_data_cards(dataset_name, link_to_source):
+    if dataset_name != "openassistant_oasst1_h2ogpt_llama2_chat":
+        return
+    #
+    assert os.path.exists("README-template.md"), "must be running this test from the data dir."
+    shutil.rmtree(dataset_name, ignore_errors=True)
+    try:
+        repo = huggingface_hub.Repository(
+            local_dir=dataset_name,
+            clone_from="h2oai/%s" % dataset_name,
+            repo_type="dataset",
+            skip_lfs_files=True,
+            token=True,
+        )
+        repo.git_pull()
+    except Exception as e:
+        print(str(e))
+        print("call 'huggingface_cli login' first and provide access token with write permission")
+    dataset = load_dataset("h2oai/%s" % dataset_name)["train"]
+
+    pd.set_option('display.max_columns', None)
+    with open("README-template.md", "r") as f:
+        content = f.read()
+        assert "<<DATASET_NAME>>" in content
+        content = content.replace("<<DATASET_NAME>>", dataset_name)
+
+        assert "<<NROWS>>" in content
+        content = content.replace("<<NROWS>>", str(dataset.num_rows))
+
+        assert "<<NCOLS>>" in content
+        content = content.replace("<<NCOLS>>", str(dataset.num_columns))
+
+        assert "<<COLNAMES>>" in content
+        content = content.replace("<<COLNAMES>>", str(dataset.column_names))
+
+        # assert "<<PREVIEW>>" in content
+        # content = content.replace("<<PREVIEW>>", str(dataset.to_pandas().iloc[:5, :]))
+
+        assert "<<SOURCE_LINK>>" in content
+        content = content.replace("<<SOURCE_LINK>>", link_to_source)
+
+        assert "<<" not in content
+        assert ">>" not in content
+
+    with open(os.path.join(dataset_name, "README.md"), "w") as f:
+        f.write(content)
+    try:
+        repo.commit("Update README.md")
+        repo.push_to_hub()
+    except Exception as e:
+        print(str(e))

data/dai_docs.valid.json

@@ -0,0 +1,101 @@
+[
+  {
+    "output": " .. _feature_store:\n\n##########################\nH2O AI Feature Store Setup\n##########################\n\nYou can use the H2O AI Feature Store to store, update, and share the features data scientists, developers, and engineers need to build AI models."
+  },
+  {
+    "output": " Note: For more information on the H2O AI Feature Store, refer to the `official documentation <https://docs.h2o.ai/feature-store/latest-stable/docs/index.html>`_."
+  },
+  {
+    "output": " To enable the Feature Store data connector, ``feature_store`` must be added to this list of data sources."
+  },
+  {
+    "output": " - ``feature_store_enable_tls``: To enable TLS communication between DAI and the Feature Store server, set this to ``true``."
+  },
+  {
+    "output": " - ``authentication_method``: The authentication method used by DAI. When enabling the Feature Store data connector, this must be set to OpenID Connect (``authentication_method=\"oidc\"``)."
+  },
+  {
+    "output": " \nUpgrading the Driverless AI Image\n~\n\n.. include:: upgrade-warning.frag\n \nUpgrading from Version 1.2.2 or Earlier\n'\n\nIt is not possible to upgrade from version 1.2.2 or earlier to the latest version."
+  },
+  {
+    "output": " Be sure to backup your data before doing this. Upgrading from Version 1.3.0 to 1.5.1\n'\n\n1. SSH into the IP address of the image instance and copy the existing experiments to a backup location:\n\n .. code-block:: bash\n\n  # Set up a directory of the previous version name\n  mkdir dai_rel_1.3.0\n\n  # Copy the data, log, license, and tmp directories as backup\n  cp -a ./data dai_rel_1.3.0/data\n  cp -a ./log dai_rel_1.3.0/log\n  cp -a ./license dai_rel_1.3.0/license\n  cp -a ./tmp dai_rel_1.3.0/tmp\n\n2."
+  },
+  {
+    "output": " Replace VERSION and BUILD below with the Driverless AI version. .. code-block:: bash\n\n   wget https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai/VERSION-BUILD/x86_64/dai-docker-ubi8-x86_64-VERSION.tar.gz\n\n3."
+  },
+  {
+    "output": " Run ``docker images`` to find the new image tag. 5. Start the Driverless AI Docker image and replace TAG below with the image tag."
+  },
+  {
+    "output": " Note: Use ``docker version`` to check which version of Docker you are using. .. tabs::\n\n   .. tab:: >= Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      docker run runtime=nvidia \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\n   .. tab:: < Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      nvidia-docker run \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\nUpgrading from version 1.5.2 or Later\n'\n\nUpgrading to versions 1.5.2 and later is no longer done via Docker."
+  },
+  {
+    "output": " Replace ``dai_NEWVERSION.deb`` below with the new Driverless AI version (for example, ``dai_1.8.4.1_amd64.deb``)."
+  },
+  {
+    "output": " You do not need to manually specify the DAI_USER or DAI_GROUP environment variables during an upgrade."
+  },
+  {
+    "output": " Driverless AI ships with CUDA 11.2.2 for GPUs, but the driver must exist in the host environment. Go to `NVIDIA download driver <https://www.nvidia.com/Download/index.aspx>`__ to get the latest NVIDIA Tesla A/T/V/P/K series drivers."
+  },
+  {
+    "output": " .. note::\n\tIf you are using K80 GPUs, the minimum required NVIDIA driver version is 450.80.02. .. code-block:: bash\n\n  # Stop Driverless AI."
+  },
+  {
+    "output": " .. _feature_store:\n\n##########################\nH2O AI Feature Store Setup\n##########################\n\nYou can use the H2O AI Feature Store to store, update, and share the features data scientists, developers, and engineers need to build AI models."
+  },
+  {
+    "output": " Note: For more information on the H2O AI Feature Store, refer to the `official documentation <https://docs.h2o.ai/feature-store/latest-stable/docs/index.html>`_. Description of relevant configuration attributes\n\n\nThe following are descriptions of the relevant configuration attributes when enabling the H2O AI Feature Store data connector:\n\n- ``enabled_file_systems``: A list of file systems you want to enable."
+  },
+  {
+    "output": " - ``feature_store_endpoint_url``: A URL that points to the Feature Store server. - ``feature_store_enable_tls``: To enable TLS communication between DAI and the Feature Store server, set this to ``true``."
+  },
+  {
+    "output": " - ``authentication_method``: The authentication method used by DAI. When enabling the Feature Store data connector, this must be set to OpenID Connect (``authentication_method=\"oidc\"``). For information on setting up OIDC Authentication in Driverless AI, see :ref:`oidc_auth`."
+  },
+  {
+    "output": " \nUpgrading the Driverless AI Image\n~\n\n.. include:: upgrade-warning.frag\n \nUpgrading from Version 1.2.2 or Earlier\n'\n\nIt is not possible to upgrade from version 1.2.2 or earlier to the latest version."
+  },
+  {
+    "output": " Be sure to backup your data before doing this. Upgrading from Version 1.3.0 to 1.5.1\n'\n\n1. SSH into the IP address of the image instance and copy the existing experiments to a backup location:\n\n .. code-block:: bash\n\n  # Set up a directory of the previous version name\n  mkdir dai_rel_1.3.0\n\n  # Copy the data, log, license, and tmp directories as backup\n  cp -a ./data dai_rel_1.3.0/data\n  cp -a ./log dai_rel_1.3.0/log\n  cp -a ./license dai_rel_1.3.0/license\n  cp -a ./tmp dai_rel_1.3.0/tmp\n\n2."
+  },
+  {
+    "output": " Replace VERSION and BUILD below with the Driverless AI version. .. code-block:: bash\n\n   wget https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai/VERSION-BUILD/x86_64/dai-docker-ubi8-x86_64-VERSION.tar.gz\n\n3."
+  },
+  {
+    "output": " Run ``docker images`` to find the new image tag. 5. Start the Driverless AI Docker image and replace TAG below with the image tag. Depending on your install version, use the ``docker run runtime=nvidia`` (>= Docker 19.03) or ``nvidia-docker`` (< Docker 19.03) command."
+  },
+  {
+    "output": " .. tabs::\n\n   .. tab:: >= Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      docker run runtime=nvidia \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\n   .. tab:: < Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      nvidia-docker run \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\nUpgrading from version 1.5.2 or Later\n'\n\nUpgrading to versions 1.5.2 and later is no longer done via Docker."
+  },
+  {
+    "output": " Replace ``dai_NEWVERSION.deb`` below with the new Driverless AI version (for example, ``dai_1.8.4.1_amd64.deb``). Note that this upgrade process inherits the service user and group from /etc/dai/User.conf and /etc/dai/Group.conf."
+  },
+  {
+    "output": " We recommend to have NVIDIA driver >= |NVIDIA-driver-ver| installed (GPU only) in your host environment for a seamless experience on all architectures, including Ampere. Driverless AI ships with CUDA 11.2.2 for GPUs, but the driver must exist in the host environment."
+  },
+  {
+    "output": " For reference on CUDA Toolkit and Minimum Required Driver Versions and CUDA Toolkit and Corresponding Driver Versions, see `here <https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html>`__ ."
+  },
+  {
+    "output": " .. _feature_store:\n\n##########################\nH2O AI Feature Store Setup\n##########################\n\nYou can use the H2O AI Feature Store to store, update, and share the features data scientists, developers, and engineers need to build AI models. This page describes how to configure Driverless AI to work with the H2O AI Feature Store. Note: For more information on the H2O AI Feature Store, refer to the `official documentation <https://docs.h2o.ai/feature-store/latest-stable/docs/index.html>`_."
+  },
+  {
+    "output": " To enable the Feature Store data connector, ``feature_store`` must be added to this list of data sources. - ``feature_store_endpoint_url``: A URL that points to the Feature Store server. - ``feature_store_enable_tls``: To enable TLS communication between DAI and the Feature Store server, set this to ``true``. - ``feature_store_access_token_scopes``: A space-separated list of access token scopes used by the Feature Store connector for authentication. - ``authentication_method``: The authentication method used by DAI."
+  },
+  {
+    "output": " \nUpgrading the Driverless AI Image\n~\n\n.. include:: upgrade-warning.frag\n \nUpgrading from Version 1.2.2 or Earlier\n'\n\nIt is not possible to upgrade from version 1.2.2 or earlier to the latest version. You have to manually remove the 1.2.2 container and then reinstall the latest Driverless AI version. Be sure to backup your data before doing this. Upgrading from Version 1.3.0 to 1.5.1\n'\n\n1. SSH into the IP address of the image instance and copy the existing experiments to a backup location:\n\n .. code-block:: bash\n\n  # Set up a directory of the previous version name\n  mkdir dai_rel_1.3.0\n\n  # Copy the data, log, license, and tmp directories as backup\n  cp -a ./data dai_rel_1.3.0/data\n  cp -a ./log dai_rel_1.3.0/log\n  cp -a ./license dai_rel_1.3.0/license\n  cp -a ./tmp dai_rel_1.3.0/tmp\n\n2."
+  },
+  {
+    "output": " Replace VERSION and BUILD below with the Driverless AI version. .. code-block:: bash\n\n   wget https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/dai/VERSION-BUILD/x86_64/dai-docker-ubi8-x86_64-VERSION.tar.gz\n\n3. Use the ``docker load`` command to load the image:\n\n .. code-block:: bash\n\n   docker load < dai-docker-ubi8-x86_64-VERSION.tar.gz\n\n4. Run ``docker images`` to find the new image tag. 5. Start the Driverless AI Docker image and replace TAG below with the image tag. Depending on your install version, use the ``docker run runtime=nvidia`` (>= Docker 19.03) or ``nvidia-docker`` (< Docker 19.03) command."
+  },
+  {
+    "output": " .. tabs::\n\n   .. tab:: >= Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      docker run runtime=nvidia \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\n   .. tab:: < Docker 19.03\n\n    .. code-block:: bash\n\n      # Start the Driverless AI Docker image\n      nvidia-docker run \\\n          pid=host \\\n          init \\\n          rm \\\n          shm-size=256m \\\n          -u `id -u`:`id -g` \\\n          -p 12345:12345 \\\n          -v `pwd`/data:/data \\\n          -v `pwd`/log:/log \\\n          -v `pwd`/license:/license \\\n          -v `pwd`/tmp:/tmp \\\n          h2oai/dai-ubi8-x86_64:TAG\n\nUpgrading from version 1.5.2 or Later\n'\n\nUpgrading to versions 1.5.2 and later is no longer done via Docker."
+  },
+  {
+    "output": " Replace ``dai_NEWVERSION.deb`` below with the new Driverless AI version (for example, ``dai_1.8.4.1_amd64.deb``). Note that this upgrade process inherits the service user and group from /etc/dai/User.conf and /etc/dai/Group.conf. You do not need to manually specify the DAI_USER or DAI_GROUP environment variables during an upgrade. We recommend to have NVIDIA driver >= |NVIDIA-driver-ver| installed (GPU only) in your host environment for a seamless experience on all architectures, including Ampere."
+  },
+  {
+    "output": " Go to `NVIDIA download driver <https://www.nvidia.com/Download/index.aspx>`__ to get the latest NVIDIA Tesla A/T/V/P/K series drivers. For reference on CUDA Toolkit and Minimum Required Driver Versions and CUDA Toolkit and Corresponding Driver Versions, see `here <https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html>`__ . .. note::\n\tIf you are using K80 GPUs, the minimum required NVIDIA driver version is 450.80.02. .. code-block:: bash\n\n  # Stop Driverless AI. sudo systemctl stop dai\n\n  # Backup your /opt/h2oai/dai/tmp directory at this time."
+  }
+]

data/dai_faq.json

@@ -0,0 +1,477 @@
+[
+  {
+    "instruction": "How is Driverless AI different than any other black box ML algorithm?",
+    "output": "Driverless AI uses many techniques (some older and some cutting-edge) for interpreting black box models including creating reason codes for every prediction the system makes. We have also created numerous open source code examples and free publications that explain these techniques. See the list below for links to these resources and for references for the interpretability techniques.Open source interpretability examples:https://github.com/jphall663/interpretable_machine_learning_with_pythonhttps://content.oreilly.com/oriole/Interpretable-machine-learning-with-Python-XGBoost-and-H2Ohttps://github.com/h2oai/mli-resourcesFree Machine Learning Interpretability publications:http://www.oreilly.com/data/free/an-introduction-to-machine-learning-interpretability.csphttp://docs.h2o.ai/driverless-ai/latest-stable/docs/booklets/MLIBooklet.pdfMachine Learning Techniques already in Driverless AI:Tree-based Variable Importance: https://web.stanford.edu/~hastie/ElemStatLearn/printings/ESLII_print12.pdfPartial Dependence: https://web.stanford.edu/~hastie/ElemStatLearn/printings/ESLII_print12.pdfLIME: http://www.kdd.org/kdd2016/papers/files/rfp0573-ribeiroA.pdfLOCO: http://www.stat.cmu.edu/~ryantibs/papers/conformal.pdfICE: https://arxiv.org/pdf/1309.6392.pdfSurrogate Models:https://papers.nips.cc/paper/1152-extracting-tree-structured-representations-of-trained-networks.pdfhttps://arxiv.org/pdf/1705.08504.pdfShapley Explanations: http://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How often do new versions come out?",
+    "output": "The frequency of major new Driverless AI releases has historically been about every two months.Installation/Upgrade/Authentication",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I change my username and password?",
+    "output": "The username and password are tied to the experiments you have created. For example, if I log in with the username/password: megan/megan and start an experiment, then I would need to log back in with the same username and password to see those experiments. The username and password, however, does not limit your access to Driverless AI. If you want to use a new user name and password, you can log in again with a new username and password, but keep in mind that you won't see your old experiments.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can Driverless AI run on CPU-only machines?",
+    "output": "Yes, Driverless AI can run on machines with CPUs only, though GPUs are recommended. Installation instructions are available for GPU and CPU systems. Refer to :ref:`before_you_begin` for more information.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I upgrade to a newer version of Driverless AI?",
+    "output": "Upgrade instructions vary depending on your environment. Refer to the installation section for your environment. Upgrade instructions are included there.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What kind of authentication is supported in Driverless AI?",
+    "output": "Driverless AI supports Client Certificate, LDAP, Local, mTLS, OpenID, none, and unvalidated (default) authentication. These can be configured by setting the appropriate environment variables in the config.toml file or by specifying the environment variables when starting Driverless AI. Refer to :ref:`dai_auth` for more information.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I automatically turn on persistence each time the GPU system reboots?",
+    "output": "For GPU machines, the sudo nvidia-persistenced --user dai command can be run after each reboot to enable persistence. For systems that have systemd, it is possible to automatically enable persistence after each reboot by removing the --no-persistence-mode flag from nvidia-persistenced.service. Before running the steps below, be sure to review the following for more information:https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemonhttps://docs.nvidia.com/deploy/driver-persistence/index.html#installationRun the following to stop the nvidia-persistenced.service:Cannot analyze code. Pygments package not found... code:: bash\n\n sudo systemctl stop nvidia-persistenced.service\nOpen the file /lib/systemd/system/nvidia-persistenced.service. This file includes a line \"ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --no-persistence-mode --verbose\".Remove the flag --no-persistence-mode from that line so that it reads:Enumerated list start value not ordinal-1: \"2\" (ordinal 2)Cannot analyze code. Pygments package not found... code:: bash\n\n ExecStart=/usr/bin/nvidia-persistenced --user nvidia-persistenced --verbose\nRun the following command to start the nvidia-persistenced.service:Enumerated list start value not ordinal-1: \"4\" (ordinal 4)Cannot analyze code. Pygments package not found... code:: bash\n\n sudo systemctl start nvidia-persistenced.service\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I start Driverless AI on a different port than 12345?",
+    "output": "No directive entry for \"tabs\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"tabs\" as canonical directive name.Unknown directive type \"tabs\"... tabs::\n   .. group-tab:: Docker Image Installs\n\n     When starting Driverless AI in Docker, the ``-p`` option specifies the port on which Driverless AI will run. Change this option in the start script if you need to run on a port other than 12345. The following example shows how to run on port 22345. (Change ``nvidia-docker run`` to ``docker-run`` if needed.) Keep in mind that `priviliged ports will require root access <https://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html>`__.\n\n     .. code-block:: bash\n        :substitutions:\n\n         nvidia-docker run \\\n         --pid=host \\\n         --init \\\n         --rm \\\n         --shm-size=256m \\\n         -u `id -u`:`id -g` \\\n         -p 22345:12345 \\\n         -v `pwd`/data:/data \\\n         -v `pwd`/log:/log \\\n         -v `pwd`/license:/license \\\n         -v `pwd`/tmp:/tmp \\\n         h2oai/dai-ubi8-x86_64:|tag|\n\n   .. group-tab:: Native Installs\n\n     To run on a port other than 12345, update the port value in the **config.toml** file. The following example shows how to run Driverless AI on port 22345. Keep in mind that `priviliged ports will require root access <https://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html>`__.\n\n     ::\n\n       # Export the Driverless AI config.toml file (or add it to ~/.bashrc)\n       export DRIVERLESS_AI_CONFIG_FILE=\u201c/config/config.toml\u201d\n\n       # IP address and port for Driverless AI HTTP server.\n       ip = \"127.0.0.1\"\n       port = 22345\n\n     Point to this updated config file when restarting Driverless AI.\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can I set up TLS/SSL on Driverless AI?",
+    "output": "Yes, Driverless AI provides configuration options that let you set up HTTPS/TLS/SSL. You will need to have your own SSL certificate, or you can create a self-signed certificate for yourself.To enable HTTPS/TLS/SSL on the Driverless AI server, add the following to the config.toml file:Cannot analyze code. Pygments package not found... code:: bash\n\n enable_https = true\n ssl_key_file = \"/etc/dai/private_key.pem\"\n ssl_crt_file = \"/etc/dai/cert.pem\"\nYou can make a self-signed certificate for testing with the following commands:Cannot analyze code. Pygments package not found... code:: bash\n\n umask 077\n openssl req -x509 -newkey rsa:4096 -keyout private_key.pem -out cert.pem -days 20 -nodes -subj '/O=Driverless AI'\n sudo chown dai:dai cert.pem private_key.pem\n sudo mv cert.pem private_key.pem /etc/dai\nTo configure specific versions of TLS/SSL, enable or disable the following settings in the config.toml file:Cannot analyze code. Pygments package not found... code:: bash\n\n       ssl_no_sslv2 = true\n       ssl_no_sslv3 = true\n       ssl_no_tlsv1 = true\n       ssl_no_tlsv1_1 = true\n       ssl_no_tlsv1_2 = false\n       ssl_no_tlsv1_3 = false\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can I set up TLS/SSL on Driverless AI in AWS?",
+    "output": "Yes, you can set up HTTPS/TLS/SSL on Driverless AI running in an AWS environment. HTTPS/TLS/SSL needs to be configured on the host machine, and the necessary ports will need to be opened on the AWS side. You will need to have your own TLS/SSL cert or you can create a self signed cert for yourself.The following is a very simple example showing how to configure HTTPS with a proxy pass to the port on the container 12345 with the keys placed in /etc/nginx/. Replace <server_name> with your server name.Cannot analyze code. Pygments package not found... code:: bash\n\n       server {\n           listen 80;\n           return 301 https://$host$request_uri;\n       }\n\n       server {\n           listen 443;\n\n           # Specify your server name here\n           server_name <server_name>;\n\n           ssl_certificate           /etc/nginx/cert.crt;\n           ssl_certificate_key       /etc/nginx/cert.key;\n           ssl on;\n           ssl_session_cache  builtin:1000  shared:SSL:10m;\n           ssl_protocols  TLSv1 TLSv1.1 TLSv1.2;\n           ssl_ciphers HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4;\n           ssl_prefer_server_ciphers on;\n\n           access_log            /var/log/nginx/dai.access.log;\n\n           location / {\n             proxy_set_header        Host $host;\n             proxy_set_header        X-Real-IP $remote_addr;\n             proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;\n             proxy_set_header        X-Forwarded-Proto $scheme;\n\n             # Fix the \u201cIt appears that your reverse proxy set up is broken\" error.\n             proxy_pass          http://localhost:12345;\n             proxy_read_timeout  90;\n\n             # Specify your server name for the redirect\n             proxy_redirect      http://localhost:12345 https://<server_name>;\n           }\n       }\nMore information about SSL for Nginx in Ubuntu 16.04 can be found here: https://www.digitalocean.com/community/tutorials/how-to-create-a-self-signed-ssl-certificate-for-nginx-in-ubuntu-16-04.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I received a \"package dai-<version>.x86_64 does not verify: no digest\" error during the installation. How can I fix this?",
+    "output": "You will recieve a \"package dai-<version>.x86_64 does not verify: no digest\" error when installing the rpm using an RPM version newer than 4.11.3. You can run the following as a workaround, replacing <version> with your DAI version:Cannot analyze code. Pygments package not found... code:: bash\n\n rpm --nodigest -i dai-<version>.x86_64.rpm\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I received a \"Must have exactly one OpenCL platform 'NVIDIA CUDA'\" error. How can I fix that?",
+    "output": "If you encounter problems with opencl errors at server time, you may see the following message:Cannot analyze code. Pygments package not found... code:: bash\n\n  2018-11-08 14:26:15,341 C:  D:452.2GB M:246.0GB 21603 ERROR  : Must have exactly one OpenCL platform 'NVIDIA CUDA', but got:\n  Platform #0: Clover\n  Platform #1: NVIDIA CUDA\n   +-- Device #0: GeForce GTX 1080 Ti\n   +-- Device #1: GeForce GTX 1080 Ti\n   +-- Device #2: GeForce GTX 1080 Ti\n\n  Uninstall all but 'NVIDIA CUDA' platform.\nFor Ubuntu, the solution is to run the following:Cannot analyze code. Pygments package not found... code:: bash\n\n  sudo apt-get remove mesa-opencl-icd\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Is it possible for multiple users to share a single Driverless AI instance?",
+    "output": "Driverless AI supports multiple users, and Driverless AI is licensed per a single named user. Therefore, in order, to have different users run experiments simultaneously, they would each need a license. Driverless AI manages the GPU(s) that it is given and ensures that different experiments from different users can run safely simultaneously and don\u2019t interfere with each other. So when two licensed users log in with different credentials, then neither of them will see the other\u2019s experiment. Similarly, if a licensed user logs in using a different set of credentials, then that user will not see any previously run experiments.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can multiple Driverless AI users share a GPU server?",
+    "output": "Yes, you can allocate multiple users in a single GPU box. For example, a single box with four GPUs can allocate that User1 has two GPUs and User2 has the other two GPUs. This is accomplished by having two separated Driverless AI instances running on the same server.There are two ways to assign specific GPUs to Driverless AI. And in the scenario with four GPUs (two GPUs allocated to two users), both of these options allow each Docker container only to see two GPUs.Use the CUDA_VISIBLE_DEVICES environment variable. In the case of Docker deployment, this will translate in passing the -e CUDA_VISIBLE_DEVICES=\"0,1\" to the nvidia-docker run command.Passing the NV_GPU option at the beginning of the nvidia-docker run command. (See example below.)Error in \"code-block\" directive:\nunknown option: \"substitutions\"... code-block:: bash\n   :substitutions:\n\n   #Team 1\n   NV_GPU='0,1' nvidia-docker run\n   --pid=host\n   --init\n   --rm\n   --shm-size=256m\n   -u id -u:id -g\n   -p port-to-team:12345\n   -e DRIVERLESS_AI_CONFIG_FILE=\"/config/config.toml\"\n   -v /data:/data\n   -v /log:/log\n   -v /license:/license\n   -v /tmp:/tmp\n   -v /config:/config\n   h2oai/dai-ubi8-x86_64:|tag|\n\n\n   #Team 2\n   NV_GPU='0,1' nvidia-docker run\n   --pid=host\n   --init\n   --rm\n   --shm-size=256m\n   -u id -u:id -g\n   -p port-to-team:12345\n   -e DRIVERLESS_AI_CONFIG_FILE=\"/config/config.toml\"\n   -v /data:/data\n   -v /log:/log\n   -v /license:/license\n   -v /tmp:/tmp\n   -v /config:/config\n   h2oai/dai-ubi8-x86_64:|tag|\nNote, however, that a Driverless AI instance expects to fully utilize and not share the GPUs that are assigned to it. Sharing a GPU with other Driverless AI instances or other running programs can result in out-of-memory issues.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I retrieve a list of Driverless AI users?",
+    "output": "A list of users can be retrieved using the Python client.Cannot analyze code. Pygments package not found... code:: bash\n\n  h2o = Client(address='http://<client_url>:12345', username='<username>', password='<password>')\n  h2o.get_users()\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Start of Driverless AI fails on the message ``Segmentation fault (core dumped)`` on Ubuntu 18/RHEL 7.6. How can I fix this?",
+    "output": "This problem is caused by the font NotoColorEmoji.ttf, which cannot be processed by the Python matplotlib library. A workaround is to disable the font by renaming it. (Do not use fontconfig because it is ignored by matplotlib.) The following will print out the command that should be executed.Cannot analyze code. Pygments package not found... code:: bash\n\n  sudo find / -name \"NotoColorEmoji.ttf\" 2>/dev/null | xargs -I{} echo sudo mv {} {}.backup\n\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Which Linux systems does Driverless AI support?",
+    "output": "Supported Linux systems include x86_64 RHEL 7, RHEL 8, CentOS 7, and CentOS 8.Data",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Is there a file size limit for datasets?",
+    "output": "For GBMs, the file size for datasets is limited by the collective CPU or GPU memory on the system, but we continue to make optimizations for getting more data into an experiment, such as using TensorFlow streaming to stream to arbitrarily large datasets.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I import CSV files that use UTF-8 encoding into Excel?",
+    "output": "Excel requires a byte order mark (BOM) to correctly identify CSV files that use UTF-8 encoding. Refer to the following FAQ entry for more information on how to use a BOM when writing CSV files with datatable.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can a byte order mark be used when writing CSV files with datatable?",
+    "output": "Yes, a byte order mark (BOM) can be used when writing CSV files with datatable by enabling datatable_bom_csv in the config.toml file when starting Driverless AI.Note: Support for UTF-8 encoding in Excel requires the use of a BOM.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Which version of Longhorn is supported by Driverless AI?",
+    "output": "Driverless AI supports Longhorn v1.1.0 or later.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Is it possible to download a transformed test dataset in Driverless AI?",
+    "output": "Yes, a transformed test dataset can be downloaded in Driverless AI. To do this, click Model Actions > Transform Dataset on the completed experiment page, then specify both a train and a test dataset to use for the transformation. The transformed test dataset is made available for download once this process is completed.Connectors",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why can't I import a folder as a file when using a data connector on Windows?",
+    "output": "If you try to use the Import Folder as File option via a data connector on Windows, the import will fail if the folder contains files that do not have file extensions. For example, if a folder contains the files file1.csv, file2.csv, file3.csv, and _SUCCESS, the function will fail due to the presence of the _SUCCESS file.Note that this only occurs if the data is sourced from a volume that is mounted from the Windows filesystem onto the Docker container via -v /path/to/windows/filesystem:/path/in/docker/container flags. This error occurs because of the difference in how files without file extensions are treated in Windows and in the Docker container (CentOS Linux).",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I get a ClassNotFoundException error when I try to select a JDBC connection. How can I fix that?",
+    "output": "The folder storing the JDBC jar file must be visible/readable by the dai process user.If you downloaded the JDBC jar file from Oracle, they may provide you with a tar.gz file that you can unpackage with the following command:Cannot analyze code. Pygments package not found... code:: bash\n\n tar --no-same-permissions --no-same-owner -xzvf <my-jdbc-driver.tar>.gz\nAlternatively you can ensure that the permissions on the file are correct in general by running the following:Cannot analyze code. Pygments package not found... code:: bash\n\n chmod -R o+rx /path/to/folder_containing_jar_file\nFinally, if you just want to check the permissions use the command ls -altr and check the final 3 values in the permissions output.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I get a org.datanucleus.exceptions.NucleusUserException: Please check your CLASSPATH and plugin specification error when attempting to connect to Hive. How can I fix that?",
+    "output": "Make sure hive-site.xml is configured in /etc/hive/conf and not in /etc/hadoop/conf.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I get a \"Permission Denied\" error during Hive import. How do I fix this?",
+    "output": "If you see the following error, your Driverless AI instance may not be able to create a temporary Hive folder due to file system permissions restrictions.Cannot analyze code. Pygments package not found... code:: bash\n\n       ERROR HiveAgent: Error during execution of query: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: Permission denied;\n       org.apache.spark.sql.AnalysisException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: Permission denied;\nTo fix this error, add the following name-value pair to your hive-site.xml file to specify the location that is accessible to Driverless AI (that is, your Driverless AI /tmp directory).Cannot analyze code. Pygments package not found... code:: bash\n\n         <property>\n           <name>hive.exec.local.scratchdir</name>\n           <value>/path/to/dai/tmp</value>\n         </property>\nRecipes",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Where can I retrieve H2O's custom recipes?",
+    "output": "H2O's custom recipes can be obtained from the official :recipes-repo:`Recipes for Driverless AI repository <https://github.com/h2oai/driverlessai-recipes/tree/>`.No role entry for \"recipes-repo\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"recipes-repo\" as canonical role name.Unknown interpreted text role \"recipes-repo\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I create my own custom recipe?",
+    "output": "Refer to the :recipes-writing:`How to Write a Recipe <https://github.com/h2oai/driverlessai-recipes/blob/>` guide for details on how to create your own custom recipe.No role entry for \"recipes-writing\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"recipes-writing\" as canonical role name.Unknown interpreted text role \"recipes-writing\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Are MOJOs supported for experiments that use custom recipes?",
+    "output": "In most cases, MOJOs will not be available for custom recipes. Unless the recipe is simple, creating the MOJO is only possible with additional MOJO runtime support. Contact [email protected] for more information about creating MOJOs for custom recipes. (Note: The Python Scoring Pipeline features full support for custom recipes.)",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I use BYOR in my airgapped installation?",
+    "output": "If your Driverless AI environment cannot access Internet and, thus, cannot access Driverless AI's \"Bring Your Own Recipes\" from GitHub, please contact H2O support. We can work with you directly to help you access recipes.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "When enabling recipes in Driverless AI, can I install Python packages from my organization's internal Python package index?",
+    "output": "Yes\u2014you can use the pip_install_options :ref:`TOML option <understanding-configs>` to specify your organization's internal Python package index as follows:No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".pip_install_options=\"['--extra-index-url', 'http://my-own-repo:port']\"For more information on the --extra-index-url <url> pip install option, refer to the official pip documentation.Experiments",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How much memory does Driverless AI require in order to run experiments?",
+    "output": "Right now, Driverless AI requires approximately 10x the size of the data in system memory.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How many columns can Driverless AI handle?",
+    "output": "Driverless AI has been tested on datasets with 10k columns. When running experiments on wide data, Driverless AI automatically checks if it is running out of memory, and if it is, it reduces the number of features until it can fit in memory. This may lead to a worse model, but Driverless AI shouldn't crash because the data is wide.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How should I use Driverless AI if I have large data?",
+    "output": "Driverless AI can handle large datasets out of the box. For very large datasets (more than 10 billion rows x columns), we recommend sampling your data for Driverless AI. Keep in mind that the goal of driverless AI is to go through many features and models to find the best modeling pipeline, and not to just train a few models on the raw data (H2O-3 is ideally suited for that case).For large datasets, the recommended steps are:Run with the recommended accuracy/time/interpretability settings first, especially accuracy <= 7Gradually increase accuracy settings to 7 and choose accuracy 9 or 10 only after observing runs with <= 7.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI detect the ID column?",
+    "output": "The ID column logic is one of the following:The column is named  'id', 'Id', 'ID' or 'iD' exactlyThe column contains a significant number of unique values (above max_relative_cardinality in the config.toml file or Max. allowed fraction of uniques for integer and categorical cols in Expert settings)",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can Driverless AI handle data with missing values/nulls?",
+    "output": "Yes, data that is imported into Driverless AI can include missing values. Feature engineering is fully aware of missing values, and missing values are treated as information - either as a special categorical level or as a special number. So for target encoding, for example, rows with a certain missing feature will belong to the same group. For Categorical Encoding where aggregations of a numeric columns are calculated for a grouped categorical column, missing values are kept. The formula for calculating the mean is the sum of non-missing values divided by the count of all non-missing values. For clustering, we impute missing values. And for frequency encoding, we count the number of rows that have a certain missing feature.The imputation strategy is as follows:XGBoost/LightGBM do not need missing value imputation and may, in fact, perform worse with any specific other strategy unless the user has a strong understanding of the data.Driverless AI automatically imputes missing values using the mean for GLM.Driverless AI provides an imputation setting for TensorFlow in the config.toml file: tf_nan_impute_value post-normalization. If you set this option to 0, then missing values will be imputed. Setting it to (for example) +5 will specify 5 standard deviations outside the distribution. The default for TensorFlow is -5, which specifies that TensorFlow will treat NAs like a missing value. We recommend that you specify 0 if the mean is better.More information is available in the Missing and Unseen Values Handling section.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI deal with categorical variables? What if an integer column should really be treated as categorical?",
+    "output": "If a column has string values, then Driverless AI will treat it as a categorical feature.  There are multiple methods for how Driverless AI converts the categorical variables to numeric.  These include:One Hot Encoding: creating dummy variables for each valueFrequency Encoding: replace category with how frequently it is seen in the dataTarget Encoding: replace category with the average target value (additional steps included to prevent overfitting)Weight of Evidence: calculate weight of evidence for each category (http://ucanalytics.com/blogs/information-value-and-weight-of-evidencebanking-case/)Driverless AI will try multiple methods for representing the column and determine which representation(s) are best.If the column has integers, Driverless AI will try treating the column as a categorical column and numeric column.  It will treat any integer column as both categorical and numeric if the number of unique values is less than 50.This is configurable in the config.toml file:Cannot analyze code. Pygments package not found... code:: bash\n\n        # Whether to treat some numerical features as categorical\n        # For instance, sometimes an integer column may not represent a numerical feature but\n        # represents different numerical codes instead.\n        num_as_cat = true\n\n        # Max number of unique values for integer/real columns to be treated as categoricals (test applies to first statistical_threshold_data_size_small rows only)\n        max_int_as_cat_uniques = 50\n(Note: Driverless AI will also check if the distribution of any numeric column differs significantly from the distribution of typical numerical data using Benford's Law.   If the column distribution does not obey Benford's Law, we will also try to treat it as categorical even if there are more than 50 unique values.)",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How are outliers handled?",
+    "output": "Outliers are not removed from the data. Instead Driverless AI finds the best way to represent data with outliers. For example, Driverless AI may find that binning a variable with outliers improves performance.For target columns, Driverless AI first determines the best representation of the column. It may find that for a target column with outliers, it is best to predict the log of the column.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "If I drop several columns from the Train dataset, will Driverless AI understand that it needs to drop the same columns from the Test dataset?",
+    "output": "If you drop columns from the training dataset, Driverless AI will do the same for the validation and test datasets (if the columns are present). There is no need for these columns because no features will be created from them.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Does Driverless AI treat numeric variables as categorical variables?",
+    "output": "In certain cases, yes. You can prevent this behavior by setting the num_as_cat variable in your installation's config.toml file to false. You can have finer grain control over this behavior by excluding the Numeric to Categorical Target Encoding Transformer and the Numeric To Categorical Weight of Evidence Transformer and their corresponding genes in your installation's config.toml file. To learn more about the config.toml file, see the :ref:`config_file` section.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Which algorithms are used in Driverless AI?",
+    "output": "Features are engineered with a proprietary stack of Kaggle-winning statistical approaches including some of the most sophisticated target encoding and likelihood estimates based on groupings, aggregations and joins, but we also employ linear models, neural nets, clustering and dimensionality reduction models and many traditional approaches such as one-hot encoding etc.On top of the engineered features, sophisticated models are fitted, including, but not limited to: XGBoost (both original XGBoost and 'lossguide' (LightGBM) mode), Decision Trees, GLM, TensorFlow (including a TensorFlow NLP recipe based on CNN Deeplearning models), RuleFit, FTRL (Follow the Regularized Leader), Isolation Forest, and Constant Models. (Refer to :ref:`supported_algorithms` for more information.) And additional algorithms can be added via :ref:`Recipes <custom-recipes>`.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".In general, GBMs are the best single-shot algorithms. Since 2006, boosting methods have proven to be the most accurate for noisy predictive modeling tasks outside of pattern recognition in images and sound (https://www.cs.cornell.edu/~caruana/ctp/ct.papers/caruana.icml06.pdf). The advent of XGBoost and Kaggle only cemented this position.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why do my selected algorithms not show up in the Experiment Preview?",
+    "output": "When changing the algorithms used via Expert Settings > Model and Expert Settings > Recipes, you may notice in the Experiment Preview that those changes are not applied. Driverless AI determines whether to include models and/or recipes based on a hierarchy of those expert settings as well as data types (numeric, categorical, text, image, etc.) and system properties (GPUs, multiple GPUs, etc.).Setting an Algorithm to \"OFF\" in Expert Settings: If an algorithm is turned OFF in Expert Settings (for example, GLM Models) when running, then that algorithm will not be included in the experiment.Algorithms Not Included from Recipes (BYOR): If an algorithm from a custom recipe is not selected for the experiment in the Include specific models option, then that algorithm will not be included in the experiment, regardless of whether that same algorithm is set to AUTO or ON on the Expert Settings > Model page.Algorithms Not Specified as \"OFF\" and Included from Recipes: If a Driverless AI algorithm is specified as either \"AUTO\" or \"ON\" and additional models are selected for the experiment in the Include specific models option, than those algorithms may or may not be included in the experiment. Driverless AI will determine the algorithms to use based on the data and experiment type.To show warnings in the preview for which models were not used, set show_inapplicable_models_preview = true in config.toml",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why do my selected transformers not show up in the Experiment Preview?",
+    "output": "When changing the transformers used via Expert Settings > Transformers and Expert Settings > Recipes, you may notice in the Experiment Preview that those changes are not applied. Driverless AI determines whether to include transformers can be used based upon data types (numeric, categorical, text, image, etc.) and system properties (GPUs, multiple GPUs, etc.).Transformers Not Included from Recipes (BYOR): If a transformer from a custom recipe is not selected for the experiment in the Include specific transformers option, then that transformer will not be included in the experiment.To show warnings in the preview for which models were not used, set show_inapplicable_transformers_preview = true in config.toml",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can we turn on TensorFlow Neural Networks so they are evaluated?",
+    "output": "Neural networks are considered by Driverless AI, although they may not be evaluated by default.  To ensure that neural networks are tried, you can turn on TensorFlow in the Expert Settings:Once you have set TensorFlow to ON.  You should see the Experiment Preview on the left hand side change and mention that it will evaluate TensorFlow models:We recommend using TensorFlow neural networks if you have a multinomial use case with more than 5 unique values.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Does Driverless AI standardize the data?",
+    "output": "Driverless AI will automatically do variable standardization for certain algorithms.  For example, with Linear Models and Neural Networks, the data is automatically standardized. For decision tree algorithms, however, we do not perform standardization because these algorithms do not benefit from standardization.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What objective function is used in XGBoost?",
+    "output": "The objective function used in XGBoost is:reg:squarederror and a custom absolute error objective function for regressionbinary:logistic or multi:softprob for classificationThe objective function does not change depending on the scorer chosen. The scorer influences parameter tuning only. For regression, Tweedie, Gamma, and Poisson regression objectives are supported.More information on the XGBoost instantiations can be found in the logs and in the model summary, both of which can be downloaded from the GUI or found in the /tmp/h2oai_experiment_<name>/ folder on the server.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Does Driverless AI perform internal or external validation?",
+    "output": "Driverless AI does internal validation when only training data is provided. It does external validation when training and validation data are provided. In either scenario, the validation data is used for all parameter tuning (models and features), not just for feature selection. Parameter tuning includes target transformation, model selection, feature engineering, feature selection, stacking, etc.Specifically:Internal validation (only training data given):Ideal when data is either close to i.i.d., or for time-series problemsInternal holdouts are used for parameter tuning, with temporal causality for time-series problemsWill do the full spectrum from single holdout split to 5-fold CV, depending on accuracy settingsNo need to split training data manuallyFinal models are trained using CV on the training dataExternal validation (training + validation data given):Ideal when there\u2019s some amount of drift in the data, and the validation set mimics the test set data better than the training dataNo training data wasted during training because training data not used for parameter tuningValidation data is used only for parameter tuning, and is not part of training dataNo CV possible because we explicitly do not want to overfit on the training dataNot allowed for time-series problems (see Time Series FAQ section that follows)Tip: If you want both training and validation data to be used for parameter tuning (the training process), just concatenate the datasets together and turn them both into training data for the \u201cinternal validation\u201d method.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI prevent overfitting?",
+    "output": "Driverless AI performs a number of checks to prevent overfitting. For example, during certain transformations, Driverless AI calculates the average on out-of-fold data using cross validation. Driverless AI also performs early stopping for every model built, ensuring that the model build will stop when it ceases to improve on holdout data. And additional steps to prevent overfitting include checking for i.i.d. and avoiding leakage during feature engineering.A blog post describing Driverless AI overfitting protection in greater detail is available here: https://www.h2o.ai/blog/driverless-ai-prevents-overfitting-leakage/.More aggressive overfit protection can be enabled by setting lock_ga_to_final_trees=true to true or using recipe='more_overfit_protection' and fixed_only_first_fold_model='true' and for time-series experiments allow_stabilize_varimp_for_ts=true.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI avoid the multiple hypothesis (MH) problem?",
+    "output": "Driverless AI uses a variant of the reusable holdout technique to address the multiple hypothesis problem. Refer to https://pdfs.semanticscholar.org/25fe/96591144f4af3d8f8f79c95b37f415e5bb75.pdf for more information.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI suggest the experiment settings?",
+    "output": "When you run an experiment on a dataset, the experiment settings (Accuracy, Time, and Interpretability) are automatically suggested by Driverless AI. For example, Driverless AI may suggest the parameters Accuracy = 7, Time = 3, Interpretability = 6, based on your data.Driverless AI will automatically suggest experiment settings based on the number of columns and number of rows in your dataset. The settings are suggested to ensure best handling when the data is small. If the data is small, Driverless AI will suggest the settings that prevent overfitting and ensure the full dataset is utilized.If the number of rows and number of columns are each below a certain threshold, then:Accuracy will be increased up to 8.The accuracy is increased so that cross validation is done. (We don't want to \"throw away\" any data for internal validation purposes.)Interpretability will be increased up to 8.The higher the interpretability setting, the smaller the number of features in the final model.More complex features are not allowed.This prevents overfitting.Time will be decreased down to 2.There will be fewer feature engineering iterations to prevent overfitting.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What happens when I set Interpretability and Accuracy to the same number?",
+    "output": "The answer is currently that interpretability controls which features are created and what features are kept. (Also above interpretability = 6, monotonicity constraints are used in XGBoost GBM, XGBoost Dart, LightGBM, and Decision Tree models.) The accuracy refers to how hard Driverless AI then tries to make those features into the most accurate model",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can I specify the number of GPUs to use when running Driverless AI?",
+    "output": "When running an experiment, the Expert Settings let you specify the starting GPU ID for Driverless AI to use. You can also specify the maximum number of GPUs to use per model and per experiment. Refer to the :ref:`expert-settings` section for more information.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I create the simplest model in Driverless AI?",
+    "output": "To create the simplest model in Driverless AI, set the following Experiment Settings:Set Accuracy to 1. Note that this can hurt performance as a sample will be used. If necessary, adjust the knob until the preview shows no sampling.Set Time to 1.Set Interpretability to 10.Next, configure the following Expert Settings:Turn OFF all algorithms except GLM.Set GLM models to ON.Set Ensemble level to 0.Set Select target transformation of the target for regression problems to Identity.Disable Data distribution shift detection.Disable Target Encoding.Alternatively, you can set Pipeline Building Recipe to Compliant. Compliant automatically configures the following experiment and expert settings:interpretability=10 (To avoid complexity. This overrides GUI or Python client settings for Interpretability.)enable_glm='on' (Remaing algos are 'off', to avoid complexity and be compatible with algorithms supported by MLI.)num_as_cat=true: Treat some numerical features as categorical. For instance, sometimes an integer column may not represent a numerical feature but represent different numerical codes instead.fixed_ensemble_level=0: Don't use any ensemble (to avoid complexity).feature_brain_level=0: No feature brain used (to ensure every restart is identical).max_feature_interaction_depth=1: Interaction depth is set to 1 (no multi-feature interactions to avoid complexity).target_transformer=\"identity\": For regression (to avoid complexity).check_distribution_shift=\"off\": Don't use distribution shift between train, valid, and test to drop features (bit risky without fine-tuning).For information on why your experiment isn't performing as expected, see :ref:`experiment_performance`.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "When I run multiple experiments with different seeds, why do I see different scores, runtimes, and sizes on disk in the Experiments listing page?",
+    "output": "When running multiple experiments with all of the same settings except the seed, understand that a feature brain level > 0 can lead to variations in models, features, timing, and sizes on disk. (The default value is 2.) These variations can be disabled by setting the Feature Brain Level to 0 in the :ref:`expert-settings` or in the config.toml file.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".In addition, if you use a different seed for each experiment, then each experiment can be different due to the randomness in the genetic algorithm that searches for the best features and model parameters. Only if Reproducible is set with the same seed and with a feature brain level of 0 should users expect the same outcome. Once a different seed is set, the models, features, timing, and sizes on disk can all vary within the constraints set by the choices made for the experiment. (I.e., accuracy, time, interpretability, expert settings, etc., all constrain the outcome, and then a different seed can change things within those constraints.)",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why does the final model performance appear to be worse than previous iterations?",
+    "output": "There are a few things to remember:Driverless AI creates a best effort estimate of the generalization performance of the best modeling pipeline found so far.The performance estimation is always based on holdout data (data unseen by the model).If no validation dataset is provided, the training data is split internally to create internal validation holdout data (once or multiple times or cross-validation, depending on the accuracy settings).If no validation dataset is provided, for accuracy <= 7, a single holdout split is used, and a \"lucky\" or \"unlucky\" split can bias estimates for small datasets or datasets with high variance.If a validation dataset is provided, then all performance estimates are solely based on the entire validation dataset (independent of accuracy settings).All scores reported are based on bootstrapped-based statistical methods and come with error bars that represent a range of estimate uncertainty.After the final iteration, a best final model is trained on a final set of engineered features. Depending on accuracy settings, a more accurate estimation of generalization performance may be done using cross-validation. Also, the final model may be a stacked ensemble consisting of multiple base models, which generally leads to better performance. Consequently, in rare cases, the difference in performance estimation method can lead to the final model's estimated performance seeming poorer than those from previous iterations. (i.e., The final model's estimated score is significantly worse than the last iteration score and error bars don't overlap.) In that case, it is very likely that the final model performance estimation is more accurate, and the prior estimates were biased due to a \"lucky\" split. To confirm this, you can re-run the experiment multiple times (without setting the reproducible flag).If you would like to minimize the likelihood of the final model performance appearing worse than previous iterations, here are some recommendations:Increase accuracy settingsProvide a validation datasetProvide more data",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I find features that may be causing data leakages in my Driverless AI model?",
+    "output": "To find original features that are causing leakage, have a look at features_orig.txt in the experiment summary download. Features causing leakage will have high importance there. To get a hint at derived features that might be causing leakage, create a new experiment with dials set to 2/2/8, and run the new experiment on your data with all your features and response. Then analyze the top 1-2 features in the model variable importance. They are likely the main contributors to data leakage if it is occurring.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I see the performance metrics on the test data?",
+    "output": "As long as you provide a target column in the test set, Driverless AI will show the best estimate of the final model's performance on the test set at the end of the experiment. The test set is never used to tune parameters (unlike to what Kagglers often do), so this is purely a convenience. Of course, you can still make test set predictions and compute your own metrics using a method of your choice.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I see all the performance metrics possible for my experiment?",
+    "output": "At the end of the experiment, the model's estimated performance on all provided datasets with a target column is printed in the experiment logs. For example, for the test set:Cannot analyze code. Pygments package not found... code:: bash\n\n       Final scores on test (external holdout) +/- stddev:\n                      GINI = 0.87794 +/- 0.035305 (more is better)\n                       MCC = 0.71124 +/- 0.043232 (more is better)\n                       F05 = 0.79175 +/- 0.04209 (more is better)\n                        F1 = 0.75823 +/- 0.038675 (more is better)\n                        F2 = 0.82752 +/- 0.03604 (more is better)\n                  ACCURACY = 0.91513 +/- 0.011975 (more is better)\n                   LOGLOSS = 0.28429 +/- 0.016682 (less is better)\n                     AUCPR = 0.79074 +/- 0.046223 (more is better)\n        optimized: AUC = 0.93386 +/- 0.018856 (more is better)\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What if my training/validation and testing data sets come from different distributions?",
+    "output": "In general, Driverless AI uses training data to engineer features and train models and validation data to tune all parameters. If no external validation data is given, the training data is used to create internal holdouts. The way holdouts are created internally depends on whether there is a strong time dependence, see the point below. If the data has no obvious time dependency (e.g., if there is no time column neither implicit or explicit), or if the data can be sorted arbitrarily and it won't affect the outcome (e.g., Iris data, predicting flower species from measurements), and if the test dataset is different (e.g., new flowers or only large flowers), then the model performance on validation (either internal or external) as measured during training won't be achieved during final testing due to the obvious inability of the model to generalize.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Does Driverless AI handle weighted data?",
+    "output": "Yes. You can optionally provide an extra weight column in your training (and validation) data with non-negative observation weights. This can be useful to implement domain-specific effects such as exponential weighting in time or class weights. All of our algorithms and metrics in Driverless AI support observation weights, but note that estimated likelihoods can be skewed as a consequence.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI handle fold assignments for weighted data?",
+    "output": "Currently, Driverless AI does not take the weights into account during fold creation, but you can provide a fold column to enforce your own grouping, i.e., to keep rows that belong to the same group together (either in train or valid). The fold column has to be a categorical column (integers ok) that assigns a group ID to each row. (It needs to have at least 5 groups because we do up to 5-fold CV.)",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why do I see that adding new features to a dataset deteriorates the performance of the model?",
+    "output": "You may notice that after adding one or more new features to a dataset, it deteriorates the performance of the Driverless AI model. In Driverless AI, the feature engineering sequence is fairly random and may end up not doing same things with original features if you restart entirely fresh with new columns.Beginning in Driverless AI v1.4.0, you now have the option to Restart from Last Checkpoint. This lets you pull in a new dataset with more columns, and Driverless AI will more iteratively take advantage of the new columns.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI handle imbalanced data for binary classification experiments?",
+    "output": "If you have data that is imbalanced, a binary imbalanced model can help to improve scoring with a variety of imbalanced sampling methods. An imbalanced model is able to take advantage of most (or even all) of the imbalanced dataset's positive values during sampling, while a regular model significantly limits the population of positive values. Imbalanced models, however, take more time to make predictions, and they are not always more accurate than regular models. We still recommend that you try using an imbalanced model if your data is imbalanced to see if scoring is improved over a regular model. Note that this information only applies to binary models.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How is feature importance calculated in Driverless AI?",
+    "output": "For most models, such as XGBoost or LightGBM models, Driverless AI uses normalized information gain to calculate feature importance. Other estimates of importance are sometimes used for certain models.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I want to have only one LightGBM model in the final pipeline. How can I do this?",
+    "output": "You can do this by using :ref:`ensemble-levels`. To change the ensemble level, use the Ensemble Level for Final Modeling Pipeline expert setting (fixed_ensemble_level in the config.toml), which is located in the Model tab. If you want a single model, use level 0. If you are okay with using the same model with hyperparameters but trained with multiple cross validation folds, then use level 1.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".To use only one model type, use the Include Specific Models expert setting, which is located in the Recipes tab.For more information, see :ref:`ensemble-learning-in-dai`.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".Setting fixed_ensemble_level = 0 returns a single model trained on one hundred percent of the data, not just a single model type with CV.When the Cross-validate Single Final Model expert setting is enabled (default), the single model with fixed_ensemble_level = 0 has the optimal number of trees because it is tuned with CV. Disabling this setting is not recommended when fixed_ensemble_level = 0.<img src=\"_static/ensemble_level_for_final.gif\" alt=\"Ensemble level for final modeling pipeline expert setting\" data-linktype=\"relative-path\">",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "I want to have only one LightGBM model and no FE. How can I do this?",
+    "output": "You can do this by additionally limiting the set of allowed transformations to just the OriginalTransformer, which leaves numeric features in their original form and drops all non-numeric features. To include or exclude specific transformers in your Driverless AI environment, use the Include Specific Transformers expert setting (included_transformers in the config.toml), which is located in the Recipes tab. You can also set the Feature Engineering Effort expert setting (feature_engineering_effort in the config.toml) to 0 to achieve the same effect.For more information, see :ref:`Transformations`.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".<img src=\"_static/include_specific_transformers.gif\" alt=\"Include specific transformers expert setting\" data-linktype=\"relative-path\">",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What is fast approximation in Driverless AI?",
+    "output": "Fast approximation is available for both regular and Shapley predictions. It is enabled by default for MLI / AutoDoc and turned off by default for other clients. The extent of approximation can be fully configured or turned off with the fast approximation expert settings. Enabling fast approximation can result in a significant speedup for large prediction tasks like the creation of partial dependence plots and other MLI-related tasks.The following is a list of expert settings that can be used to configure fast approximation.Regular predictions::ref:`fast-approx-trees`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".:ref:`fast-approx-one-fold`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".:ref:`fast-approx-one-model`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".Shapley predictions::ref:`fast-approx-trees-shap`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".:ref:`fast-approx-one-fold-shap`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".:ref:`fast-approx-one-model-shap`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".MLI::ref:`mli_fast_approx <mli-fast-approx-speed-up>`No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "When should fast approximation be turned off?",
+    "output": "In situations where a more detailed partial dependence plot or interpretation is required, you may want to disable fast approximation.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why does the confusion matrix sometimes show decimals instead of whole numbers?",
+    "output": "Fractional confusion matrix values most commonly arise as a consequence of the averaging of confusion matrices across cross-validation fold splits or across repeated fold splits, but the same can also happen for non-integer observation weights.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Is data sampling for multiclass use cases supported?",
+    "output": "Data sampling for multiclass use cases is not currently supported. However, it is possible to approximate the data sampling approach by adding more weight in order to penalize rare classes. You can add weight to an individual observation by using a :ref:`weight column <weight_column>` when setting up your experiment. You can also enable LightGBM multiclass balancing by setting the enable_lightgbm_multiclass_balancing configuration setting to on, which enables automatic class weighting for imbalanced multiclass problems.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".Feature Transformations",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Where can I get details of the various transformations performed in an experiment?",
+    "output": "Download the experiment's log .zip file from the GUI. This zip file includes summary information, log information, and a gene_summary.txt file with details of the transformations used in the experiment. Specifically, there is a details folder with all subprocess logs.On the server, the experiment specific files are inside the /tmp/h2oai_experiment_<name>/ folder after the experiment completes, particularly h2oai_experiment_logs_<name>.zip and h2oai_experiment_summary_<name>.zip.Predictions",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I download the predictions onto the machine where Driverless AI is running?",
+    "output": "When you select Score on Another Dataset, the predictions will automatically be stored on the machine where Driverless AI is running. They will be saved in the following locations (and can be opened again by Driverless AI, both for .csv and .bin):Training Data Predictions: tmp/h2oai_experiment_<name>/train_preds.csv (also saved as .bin)Testing Data Predictions: tmp/h2oai_experiment_<name>/test_preds.csv (also saved as .bin)New Data Predictions: tmp/h2oai_experiment_<name>/automatically_generated_name.csv. Note that the automatically generated name will match the name of the file downloaded to your local computer.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why are predicted probabilities not available when I run an experiment without ensembling?",
+    "output": "When Driverless AI provides pre-computed predictions after completing an experiment, it uses only those parts of the modeling pipeline that were not trained on the particular rows for which the predictions are made. This means that Driverless AI needs holdout data in order to create predictions, such as validation or test sets, where the model is trained on training data only. In the case of ensembles, Driverless AI uses cross-validation to generate holdout folds on the training data, so we are able to provide out-of-fold estimates for every row in the training data and, hence, can also provide training holdout predictions (that will provide a good estimate of generalization performance). In the case of a single model, though, that is trained on 100% of the training data. There is no way to create unbiased estimates for any row in the training data. While DAI uses an internal validation dataset, this is a re-usable holdout, and therefore will not contain holdout predictions for the full training dataset. You need cross-validation in order to get out-of-fold estimates, and then that's not a single model anymore. If you want to still get predictions for the training data for a single model, then you have to use the scoring API to create predictions on the training set. From the GUI, this can be done using the Score on Another Dataset button for a completed experiment. Note, though, that the results will likely be overly optimistic, too good to be true, and virtually useless.Deployment",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What drives the size of a MOJO?",
+    "output": "The size of the MOJO is based on the complexity of the final modeling pipeline (i.e., feature engineering and models). One of the biggest factors is the amount of higher-order interactions between features, especially target encoding and related features, which have to store lookup tables for all possible combinations observed in the training data. You can reduce the amount of these transformations by reducing the value of Max. feature interaction depth and/or Feature engineering effort under Expert Settings, or by increasing the interpretability settings for the experiment. Ensembles also contribute to the final modeling pipeline's complexity as each model has its own pipeline. Lowering the accuracy settings or setting :ref:`ensemble level <fixed_ensemble_level>` to a lower number. The number of features Max. pipeline features also affects the MOJO size. Text transformers are pretty bulky as well and can add to the MOJO size.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".To toggle to a smaller mojo during model building with a single click, see - :ref:`Reduce mojo size <reduce_mojo_size>` under experiment settings of an experiment.No role entry for \"ref\" in module \"docutils.parsers.rst.languages.en\".\nTrying \"ref\" as canonical role name.Unknown interpreted text role \"ref\".",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Are MOJOs thread safe?",
+    "output": "Yes, all Driverless AI MOJOs are thread safe.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Running the scoring pipeline for my MOJO is taking several hours. How can I get this to run faster?",
+    "output": "When running example.sh, Driverless AI implements a memory setting, which is suitable for most use cases. For very large models, however, it may be necessary to increase the memory limit when running the Java application for data transformation. This can be done using the -Xmx25g parameter. For example:Cannot analyze code. Pygments package not found... code:: bash\n\n  java -Xmx25g -Dai.h2o.mojos.runtime.license.file=license.sig -cp mojo2-runtime.jar ai.h2o.mojos.ExecuteMojo pipeline.mojo example.csv\n",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why have I encountered a \"Best Score is not finite\" error?",
+    "output": "Driverless AI uses 32-bit floats by default. You may encounter this error if your data value exceeds 1E38 or if you are resolving more than 1 part in 10 million. You can resolve this error using one of the following methods:Enable the Force 64-bit Precision option in the experiment's Expert Settings.orSet data_precision=\"float64\" and transformer_precision=\"float64\" in config.toml.Time Series",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What if my data has a time dependency?",
+    "output": "If you know that your data has a strong time dependency, select a time column before starting the experiment. The time column must be in a Datetime format that can be parsed by pandas, such as \"2017-11-06 14:32:21\", \"Monday, June 18, 2012\" or \"Jun 18 2018 14:34:00\" etc., or contain only integers.If you are unsure about the strength of the time dependency, run two experiments: One with time column set to \"[OFF]\" and one with time column set to \"[AUTO]\" (or pick a time column yourself).",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What is a lag, and why does it help?",
+    "output": "A lag is a feature value from a previous point in time. Lags are useful to take advantage of the fact that the current (unknown) target value is often correlated with previous (known) target values. Hence, they can better capture target patterns along the time axis.Why can't I specify a validation data set for time-series problems? Why do you look at the test set for time-series problemsThe problem with validation vs test in the time series setting is that there is only one valid way to define the split. If a test set is given, its length in time defines the validation split and the validation data has to be part of train. Otherwise the time-series validation won't be useful.For instance: Let's assume we have train = [1,2,3,4,5,6,7,8,9,10] and test = [12,13], where integers define time periods (e.g., weeks). For this example, the most natural train/valid split that mimics the test scenario would be: train = [1,2,3,4,5,6,7] and valid = [9,10], and month 8 is not included in the training set to allow for a gap. Note that we will look at the start time and the duration of the test set only (if provided), and not at the contents of the test data (neither features nor target). If the user provides validation = [8,9,10] instead of test data, then this could lead to inferior validation strategy and worse generalization. Hence, we use the user-given test set only to create the optimal internal train/validation splits. If no test set is provided, the user can provide the length of the test set (in periods), the length of the train/test gap (in periods) and the length of the period itself (in seconds).",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why does the gap between train and test matter? Is it because of creating the lag features on the test set?",
+    "output": "Taking the gap into account is necessary in order to avoid too optimistic estimates of the true error and to avoid creating history-based features like lags for the training and validation data (which cannot be created for the test data due to the missing information).",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "In regards to applying the target lags to different subsets of the time group columns, are you saying Driverless AI perform auto-correlation at \"levels\" of the time series? For example, consider the Walmart dataset where I have Store and Dept (and my target is Weekly Sales). Are you saying that Driverless AI checks for auto-correlation in Weekly Sales based on just Store, just Dept, and both Store and Dept?",
+    "output": "Currently, auto-correlation is only applied on the detected superkey (entire TGC) of the training dataset relation at the very beginning. It's used to rank potential lag-sizes, with the goal to prune the search space for the GA optimization process, which is responsible for selecting the lag features.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does Driverless AI detect the time period?",
+    "output": "Driverless AI treats each time series as a function with some frequency 1/ns. The actual value is estimated by the median of time deltas across maximal length TGC subgroups. The chosen SI unit minimizes the distance to all available SI units.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What is the logic behind the selectable numbers for forecast horizon length?",
+    "output": "The shown forecast horizon options are based on quantiles of valid splits. This is necessary because Driverless AI cannot display all possible options in general.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Assume that in my Walmart dataset, all stores provided data at the week level, but one store provided data at the day level. What would Driverless AI do?",
+    "output": "Driverless AI would still assume \"weekly data\" in this case because the majority of stores are yielding this property. The \"daily\" store would be resampled to the detected overall frequency.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Assume that in my Walmart dataset, all stores and departments provided data at the weekly level, but one department in a specific store provided weekly sales on a bi-weekly basis (every two weeks). What would Driverless AI do?",
+    "output": "That's similar to having missing data. Due to proper resampling, Driverless AI can handle this without any issues.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Why does the number of weeks that you want to start predicting matter?",
+    "output": "That's an option to provide a train-test gap if there is no test data is available. That is to say, \"I don't have my test data yet, but I know it will have a gap to train of x.\"",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Are the scoring components of time series sensitive to the order in which new pieces of data arrive? I.e., is each row independent at scoring time, or is there a real-time windowing effect in the scoring pieces?",
+    "output": "Each row is independent at scoring time.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What happens if the user, at predict time, gives a row with a time value that is too small or too large?",
+    "output": "Internally, \"out-of bounds\" time values are encoded with special values. The samples will still be scored, but the predictions won't be trustworthy.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What's the minimum data size for a time series recipe?",
+    "output": "We recommended that you have around 10,000 validation samples in order to get a reliable estimate of the true error. The time series recipe can still be applied for smaller data, but the validation error might be inaccurate.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How long must the training data be compared to the test data?",
+    "output": "At a minimum, the training data has to be at least twice as long as the test data along the time axis. However, we recommended that the training data is at least three times as long as the test data.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How does the time series recipe deal with missing values?",
+    "output": "Missing values will be converted to a special value, which is different from any non-missing feature value. Explicit imputation techniques won't be applied.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "Can the time information be distributed across multiple columns in the input data (such as [year, day, month]?",
+    "output": "Currently Driverless AI requires the data to have the time stamps given in a single column. Driverless AI will create additional time features like [year, day, month] on its own, if they turn out to be useful.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What type of modeling approach does Driverless AI use for time series?",
+    "output": "Driverless AI combines the creation of history-based features like lags, moving averages etc. with the modeling techniques, which are also applied for i.i.d. data. The primary model of choice is XGBoost.",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "What's the idea behind exponential weighting of moving averages?",
+    "output": "Exponential weighting accounts for the possibility that more recent observations are better suited to explain the present than older observations.Logging",
+    "prompt_type": "human_bot"
+  },
+  {
+    "instruction": "How can I reduce the size of the Audit Logger?",
+    "output": "An Audit Logger file is created every day that Driverless AI is in use. The audit_log_retention_period config variable lets you specify the number of days, after which the audit.log will be overwritten. This option defaults to 5 days, which means that Driverless AI will maintain Audit Logger files for the last 5 days, and audit.log files older than 5 days are removed and replaced with newer log files. When this option is set to 0, the audit.log file will not be overwritten.",
+    "prompt_type": "human_bot"
+  }
+]