add crowd counting demo

.gitignore

@@ -0,0 +1,3 @@
+crowd-counting-env/
+testing.ipynb
+__pycache__/

README.md

@@ -1,13 +1,2 @@
----
-title: Video Crowd Counting
-emoji: 🚀
-colorFrom: pink
-colorTo: red
-sdk: gradio
-sdk_version: 4.21.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# crowd-counting-demo
+ 

app.py

@@ -0,0 +1,25 @@
+import gradio as gr
+from crowd_counter import CrowdCounter
+from PIL import Image
+import cv2
+
+crowd_counter_engine = CrowdCounter()
+
+def predict(inp):
+    inp = Image.fromarray(inp.astype('uint8'), 'RGB')
+    response = crowd_counter_engine.inference(inp)
+    crowd_count = response[0]
+    pred_img= response[1]
+    return cv2.cvtColor(pred_img, cv2.COLOR_BGR2RGB), crowd_count
+
+title = "Crowd Counter Demo"
+desc = "A Demo of Proposal Point Prediction for Crowd Counting - Powered by P2PNet"
+examples = [
+    ["images/img-1.jpg"],
+    ["images/img-2.jpg"],
+    ["images/img-3.jpg"],
+]
+inputs = gr.inputs.Image(label="Image of Crowd")
+outputs = [gr.outputs.Image(label="Proposal Points Prediction",type = "numpy"), gr.outputs.Label(label="Predicted Count",type = "numpy")]
+gr.Interface(fn=predict, inputs=inputs, outputs=outputs, title=title, description=desc, examples=examples,
+             allow_flagging=False).launch()

crowd_counter/LICENSE

@@ -0,0 +1,1401 @@
+Tencent is pleased to support the open source community by making P2PNet available. 
+
+Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved. The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) THL A29 Limited.
+
+P2PNet is licensed under the following License, except for the third-party components listed below. 
+
+License for P2PNet:
+--------------------------------------------------------------------
+Redistribution and use in source and binary forms, with or without modification, are permitted provided 
+that the following conditions are met:
+
+1. Use in source and binary forms shall only be for the purpose of academic research.
+
+2. Redistributions of source code must retain the above copyright notice, this list of conditions and the 
+    following disclaimer.
+
+3. Redistributions in binary form must reproduce the above copyright notice,  this list of conditions and 
+    the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+4. Neither the name of the copyright holder nor the names of its contributors may be used to endorse 
+   or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Other dependencies and licenses:
+
+
+Open Source Software Licensed under the BSD 3-Clause License:
+--------------------------------------------------------------------
+1. Pytorch
+Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
+Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
+Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
+Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+Copyright (c) 2011-2013 NYU                      (Clement Farabet)
+Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
+Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+
+2. torchvision
+Copyright (c) Soumith Chintala 2016, 
+All rights reserved.
+
+3. opencv-python
+Copyright (C) 2000-2020, Intel Corporation, all rights reserved.
+Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
+Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.
+Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.
+Copyright (C) 2015-2016, Itseez Inc., all rights reserved.
+Copyright (C) 2019-2020, Xperience AI, all rights reserved.
+Third party copyrights are property of their respective owners.
+
+
+Terms of the BSD 3-Clause License:
+--------------------------------------------------------------------
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+Open Source Software Licensed under the PIL Software License:
+--------------------------------------------------------------------
+1. Pillow
+Copyright © 2010-2020 by Alex Clark and contributors
+
+
+Terms of the PIL Software License:
+--------------------------------------------------------------------
+By obtaining, using, and/or copying this software and/or its associated documentation, you agree that you have read, understood, and will comply with the following terms and conditions:
+
+Permission to use, copy, modify, and distribute this software and its associated documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appears in all copies, and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Secret Labs AB or the author not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission.
+
+SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+
+Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein:
+--------------------------------------------------------------------
+1. numpy
+Copyright (c) 2005-2020, NumPy Developers.
+All rights reserved.
+
+
+A copy of the BSD 3-Clause License is included in this file.
+
+The NumPy repository and source distributions bundle several libraries that are
+compatibly licensed.  We list these here.
+
+Name: Numpydoc
+Files: doc/sphinxext/numpydoc/*
+License: BSD-2-Clause
+
+Name: scipy-sphinx-theme
+Files: doc/scipy-sphinx-theme/*
+License: BSD-3-Clause AND PSF-2.0 AND Apache-2.0
+
+Name: lapack-lite
+Files: numpy/linalg/lapack_lite/*
+License: BSD-3-Clause
+
+Name: tempita
+Files: tools/npy_tempita/*
+License: MIT
+
+Name: dragon4
+Files: numpy/core/src/multiarray/dragon4.c
+License: MIT
+
+
+
+Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein:
+--------------------------------------------------------------------
+1. h5py
+Copyright (c) 2008 Andrew Collette and contributors
+All rights reserved.
+
+
+A copy of the BSD 3-Clause License is included in this file.
+
+For thirdparty hdf5:
+HDF5 (Hierarchical Data Format 5) Software Library and Utilities
+Copyright 2006-2007 by The HDF Group (THG).
+
+NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities
+Copyright 1998-2006 by the Board of Trustees of the University of Illinois.
+
+All rights reserved.
+
+Contributors: National Center for Supercomputing Applications (NCSA)
+at the University of Illinois, Fortner Software, Unidata Program
+Center (netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly
+and Mark Adler (gzip), and Digital Equipment Corporation (DEC).
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted for any purpose (including commercial
+purposes) provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright
+notice, this list of conditions, and the following disclaimer.
+   2. Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions, and the following
+disclaimer in the documentation and/or materials provided with the
+distribution.
+   3. In addition, redistributions of modified forms of the source or
+binary code must carry prominent notices stating that the original
+code was changed and the date of the change.
+   4. All publications or advertising materials mentioning features or
+use of this software are asked, but not required, to acknowledge that
+it was developed by The HDF Group and by the National Center for
+Supercomputing Applications at the University of Illinois at
+Urbana-Champaign and credit the contributors.
+   5. Neither the name of The HDF Group, the name of the University,
+nor the name of any Contributor may be used to endorse or promote
+products derived from this software without specific prior written
+permission from THG, the University, or the Contributor, respectively.
+
+DISCLAIMER: THIS SOFTWARE IS PROVIDED BY THE HDF GROUP (THG) AND THE
+CONTRIBUTORS "AS IS" WITH NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR
+IMPLIED. In no event shall THG or the Contributors be liable for any
+damages suffered by the users arising out of the use of this software,
+even if advised of the possibility of such damage.
+
+Portions of HDF5 were developed with support from the University of
+California, Lawrence Livermore National Laboratory (UC LLNL). The
+following statement applies to those portions of the product and must
+be retained in any redistribution of source code, binaries,
+documentation, and/or accompanying materials:
+
+This work was partially produced at the University of California,
+Lawrence Livermore National Laboratory (UC LLNL) under contract
+no. W-7405-ENG-48 (Contract 48) between the U.S. Department of Energy
+(DOE) and The Regents of the University of California (University) for
+the operation of UC LLNL.
+
+DISCLAIMER: This work was prepared as an account of work sponsored by
+an agency of the United States Government. Neither the United States
+Government nor the University of California nor any of their
+employees, makes any warranty, express or implied, or assumes any
+liability or responsibility for the accuracy, completeness, or
+usefulness of any information, apparatus, product, or process
+disclosed, or represents that its use would not infringe privately-
+owned rights. Reference herein to any specific commercial products,
+process, or service by trade name, trademark, manufacturer, or
+otherwise, does not necessarily constitute or imply its endorsement,
+recommendation, or favoring by the United States Government or the
+University of California. The views and opinions of authors expressed
+herein do not necessarily state or reflect those of the United States
+Government or the University of California, and shall not be used for
+advertising or product endorsement purposes.
+
+For third-party pytables:
+Copyright Notice and Statement for PyTables Software Library and Utilities:
+
+Copyright (c) 2002, 2003, 2004  Francesc Altet
+Copyright (c) 2005, 2006, 2007  Carabos Coop. V.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+a. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+b. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+c. Neither the name of the Carabos Coop. V. nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+For third-party python:
+Python license
+==============
+
+#. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and
+   the Individual or Organization ("Licensee") accessing and otherwise using Python
+   Python 2.7.5 software in source or binary form and its associated documentation.
+
+#. Subject to the terms and conditions of this License Agreement, PSF hereby
+   grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+   analyze, test, perform and/or display publicly, prepare derivative works,
+   distribute, and otherwise use Python Python 2.7.5 alone or in any derivative
+   version, provided, however, that PSF's License Agreement and PSF's notice of
+   copyright, i.e., "Copyright 2001-2013 Python Software Foundation; All Rights
+   Reserved" are retained in Python Python 2.7.5 alone or in any derivative version
+   prepared by Licensee.
+
+#. In the event Licensee prepares a derivative work that is based on or
+   incorporates Python Python 2.7.5 or any part thereof, and wants to make the
+   derivative work available to others as provided herein, then Licensee hereby
+   agrees to include in any such work a brief summary of the changes made to Python
+   Python 2.7.5.
+
+#. PSF is making Python Python 2.7.5 available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY WAY OF
+   EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR
+   WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE
+   USE OF PYTHON Python 2.7.5 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+#. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON Python 2.7.5
+   FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
+   MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON Python 2.7.5, OR ANY DERIVATIVE
+   THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+#. This License Agreement will automatically terminate upon a material breach of
+   its terms and conditions.
+
+#. Nothing in this License Agreement shall be deemed to create any relationship
+   of agency, partnership, or joint venture between PSF and Licensee.  This License
+   Agreement does not grant permission to use PSF trademarks or trade name in a
+   trademark sense to endorse or promote products or services of Licensee, or any
+   third party.
+
+#. By copying, installing or otherwise using Python Python 2.7.5, Licensee agrees
+   to be bound by the terms and conditions of this License Agreement.
+
+For third-party stdint:
+Copyright (c) 2006-2008 Alexander Chemeris
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+  3. The name of the author may be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein:
+--------------------------------------------------------------------
+1. scipy
+Copyright (c) 2001-2002 Enthought, Inc.  2003-2019, SciPy Developers.
+All rights reserved.
+
+
+A copy of the BSD 3-Clause License is included in this file.
+
+The SciPy repository and source distributions bundle a number of libraries that
+are compatibly licensed.  We list these here.
+
+Name: Numpydoc
+Files: doc/sphinxext/numpydoc/*
+License: 2-clause BSD
+  For details, see doc/sphinxext/LICENSE.txt
+
+Name: scipy-sphinx-theme
+Files: doc/scipy-sphinx-theme/*
+License: 3-clause BSD, PSF and Apache 2.0
+  For details, see doc/sphinxext/LICENSE.txt
+
+Name: Decorator
+Files: scipy/_lib/decorator.py
+License: 2-clause BSD
+  For details, see the header inside scipy/_lib/decorator.py
+
+Name: ID
+Files: scipy/linalg/src/id_dist/*
+License: 3-clause BSD
+  For details, see scipy/linalg/src/id_dist/doc/doc.tex
+
+Name: L-BFGS-B
+Files: scipy/optimize/lbfgsb/*
+License: BSD license
+  For details, see scipy/optimize/lbfgsb/README
+
+Name: SuperLU
+Files: scipy/sparse/linalg/dsolve/SuperLU/*
+License: 3-clause BSD
+  For details, see scipy/sparse/linalg/dsolve/SuperLU/License.txt
+
+Name: ARPACK
+Files: scipy/sparse/linalg/eigen/arpack/ARPACK/*
+License: 3-clause BSD
+  For details, see scipy/sparse/linalg/eigen/arpack/ARPACK/COPYING
+
+Name: Qhull
+Files: scipy/spatial/qhull/*
+License: Qhull license (BSD-like)
+  For details, see scipy/spatial/qhull/COPYING.txt
+
+Name: Cephes
+Files: scipy/special/cephes/*
+License: 3-clause BSD
+  Distributed under 3-clause BSD license with permission from the author,
+  see https://lists.debian.org/debian-legal/2004/12/msg00295.html
+
+  Cephes Math Library Release 2.8:  June, 2000
+  Copyright 1984, 1995, 2000 by Stephen L. Moshier
+
+  This software is derived from the Cephes Math Library and is
+  incorporated herein by permission of the author.
+
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+      * Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+      * Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+      * Neither the name of the <organization> nor the
+        names of its contributors may be used to endorse or promote products
+        derived from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Name: Faddeeva
+Files: scipy/special/Faddeeva.*
+License: MIT
+  Copyright (c) 2012 Massachusetts Institute of Technology
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Name: qd
+Files: scipy/special/cephes/dd_*.[ch]
+License: modified BSD license ("BSD-LBNL-License.doc")
+  This work was supported by the Director, Office of Science, Division
+  of Mathematical, Information, and Computational Sciences of the
+  U.S. Department of Energy under contract numbers DE-AC03-76SF00098 and
+  DE-AC02-05CH11231.
+
+  Copyright (c) 2003-2009, The Regents of the University of California,
+  through Lawrence Berkeley National Laboratory (subject to receipt of
+  any required approvals from U.S. Dept. of Energy) All rights reserved.
+
+  1. Redistribution and use in source and binary forms, with or
+  without modification, are permitted provided that the following
+  conditions are met:
+
+  (1) Redistributions of source code must retain the copyright
+  notice, this list of conditions and the following disclaimer.
+
+  (2) Redistributions in binary form must reproduce the copyright
+  notice, this list of conditions and the following disclaimer in
+  the documentation and/or other materials provided with the
+  distribution.
+
+  (3) Neither the name of the University of California, Lawrence
+  Berkeley National Laboratory, U.S. Dept. of Energy nor the names
+  of its contributors may be used to endorse or promote products
+  derived from this software without specific prior written
+  permission.
+
+  2. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  3. You are under no obligation whatsoever to provide any bug fixes,
+  patches, or upgrades to the features, functionality or performance of
+  the source code ("Enhancements") to anyone; however, if you choose to
+  make your Enhancements available either publicly, or directly to
+  Lawrence Berkeley National Laboratory, without imposing a separate
+  written license agreement for such Enhancements, then you hereby grant
+  the following license: a non-exclusive, royalty-free perpetual license
+  to install, use, modify, prepare derivative works, incorporate into
+  other computer software, distribute, and sublicense such enhancements
+  or derivative works thereof, in binary and source code form.
+
+Name: pypocketfft
+Files: scipy/fft/_pocketfft/[pocketfft.h, pypocketfft.cxx]
+License: 3-Clause BSD
+  For details, see scipy/fft/_pocketfft/LICENSE.md
+
+Name: uarray
+Files: scipy/_lib/uarray/*
+License: 3-Clause BSD
+  For details, see scipy/_lib/uarray/LICENSE
+
+Name: ampgo
+Files: benchmarks/benchmarks/go_benchmark_functions/*.py
+License: MIT
+  Functions for testing global optimizers, forked from the AMPGO project,
+  https://code.google.com/archive/p/ampgo
+
+Name: pybind11
+Files: no source files are included, however pybind11 binary artifacts are
+  included with every binary build of SciPy.
+License:
+  Copyright (c) 2016 Wenzel Jakob <[email protected]>, All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice, this
+     list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright notice,
+     this list of conditions and the following disclaimer in the documentation
+     and/or other materials provided with the distribution.
+
+  3. Neither the name of the copyright holder nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+
+Open Source Software Licensed under the Specific License and Other Licenses of the Third-Party Components therein:
+--------------------------------------------------------------------
+1. matplotlib
+Copyright (c)
+2012- Matplotlib Development Team; All Rights Reserved
+
+
+Terms of the Specific License:
+--------------------------------------------------------------------
+1. This LICENSE AGREEMENT is between the Matplotlib Development Team
+("MDT"), and the Individual or Organization ("Licensee") accessing and
+otherwise using matplotlib software in source or binary form and its
+associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, MDT
+hereby grants Licensee a nonexclusive, royalty-free, world-wide license
+to reproduce, analyze, test, perform and/or display publicly, prepare
+derivative works, distribute, and otherwise use matplotlib
+alone or in any derivative version, provided, however, that MDT's
+License Agreement and MDT's notice of copyright, i.e., "Copyright (c)
+2012- Matplotlib Development Team; All Rights Reserved" are retained in
+matplotlib  alone or in any derivative version prepared by
+Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on or
+incorporates matplotlib or any part thereof, and wants to
+make the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to matplotlib .
+
+4. MDT is making matplotlib available to Licensee on an "AS
+IS" basis.  MDT MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, MDT MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB
+WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. MDT SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB
+ FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR
+LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING
+MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF
+THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between MDT and
+Licensee.  This License Agreement does not grant permission to use MDT
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using matplotlib ,
+Licensee agrees to be bound by the terms and conditions of this License
+Agreement.
+
+For third-party cmr10.pfb:
+The cmr10.pfb file is a Type-1 version of one of Knuth's Computer Modern fonts.
+It is included here as test data only, but the following license applies.
+
+Copyright (c) 1997, 2009, American Mathematical Society (http://www.ams.org).
+All Rights Reserved.
+
+"cmb10" is a Reserved Font Name for this Font Software.
+"cmbsy10" is a Reserved Font Name for this Font Software.
+"cmbsy5" is a Reserved Font Name for this Font Software.
+"cmbsy6" is a Reserved Font Name for this Font Software.
+"cmbsy7" is a Reserved Font Name for this Font Software.
+"cmbsy8" is a Reserved Font Name for this Font Software.
+"cmbsy9" is a Reserved Font Name for this Font Software.
+"cmbx10" is a Reserved Font Name for this Font Software.
+"cmbx12" is a Reserved Font Name for this Font Software.
+"cmbx5" is a Reserved Font Name for this Font Software.
+"cmbx6" is a Reserved Font Name for this Font Software.
+"cmbx7" is a Reserved Font Name for this Font Software.
+"cmbx8" is a Reserved Font Name for this Font Software.
+"cmbx9" is a Reserved Font Name for this Font Software.
+"cmbxsl10" is a Reserved Font Name for this Font Software.
+"cmbxti10" is a Reserved Font Name for this Font Software.
+"cmcsc10" is a Reserved Font Name for this Font Software.
+"cmcsc8" is a Reserved Font Name for this Font Software.
+"cmcsc9" is a Reserved Font Name for this Font Software.
+"cmdunh10" is a Reserved Font Name for this Font Software.
+"cmex10" is a Reserved Font Name for this Font Software.
+"cmex7" is a Reserved Font Name for this Font Software.
+"cmex8" is a Reserved Font Name for this Font Software.
+"cmex9" is a Reserved Font Name for this Font Software.
+"cmff10" is a Reserved Font Name for this Font Software.
+"cmfi10" is a Reserved Font Name for this Font Software.
+"cmfib8" is a Reserved Font Name for this Font Software.
+"cminch" is a Reserved Font Name for this Font Software.
+"cmitt10" is a Reserved Font Name for this Font Software.
+"cmmi10" is a Reserved Font Name for this Font Software.
+"cmmi12" is a Reserved Font Name for this Font Software.
+"cmmi5" is a Reserved Font Name for this Font Software.
+"cmmi6" is a Reserved Font Name for this Font Software.
+"cmmi7" is a Reserved Font Name for this Font Software.
+"cmmi8" is a Reserved Font Name for this Font Software.
+"cmmi9" is a Reserved Font Name for this Font Software.
+"cmmib10" is a Reserved Font Name for this Font Software.
+"cmmib5" is a Reserved Font Name for this Font Software.
+"cmmib6" is a Reserved Font Name for this Font Software.
+"cmmib7" is a Reserved Font Name for this Font Software.
+"cmmib8" is a Reserved Font Name for this Font Software.
+"cmmib9" is a Reserved Font Name for this Font Software.
+"cmr10" is a Reserved Font Name for this Font Software.
+"cmr12" is a Reserved Font Name for this Font Software.
+"cmr17" is a Reserved Font Name for this Font Software.
+"cmr5" is a Reserved Font Name for this Font Software.
+"cmr6" is a Reserved Font Name for this Font Software.
+"cmr7" is a Reserved Font Name for this Font Software.
+"cmr8" is a Reserved Font Name for this Font Software.
+"cmr9" is a Reserved Font Name for this Font Software.
+"cmsl10" is a Reserved Font Name for this Font Software.
+"cmsl12" is a Reserved Font Name for this Font Software.
+"cmsl8" is a Reserved Font Name for this Font Software.
+"cmsl9" is a Reserved Font Name for this Font Software.
+"cmsltt10" is a Reserved Font Name for this Font Software.
+"cmss10" is a Reserved Font Name for this Font Software.
+"cmss12" is a Reserved Font Name for this Font Software.
+"cmss17" is a Reserved Font Name for this Font Software.
+"cmss8" is a Reserved Font Name for this Font Software.
+"cmss9" is a Reserved Font Name for this Font Software.
+"cmssbx10" is a Reserved Font Name for this Font Software.
+"cmssdc10" is a Reserved Font Name for this Font Software.
+"cmssi10" is a Reserved Font Name for this Font Software.
+"cmssi12" is a Reserved Font Name for this Font Software.
+"cmssi17" is a Reserved Font Name for this Font Software.
+"cmssi8" is a Reserved Font Name for this Font Software.
+"cmssi9" is a Reserved Font Name for this Font Software.
+"cmssq8" is a Reserved Font Name for this Font Software.
+"cmssqi8" is a Reserved Font Name for this Font Software.
+"cmsy10" is a Reserved Font Name for this Font Software.
+"cmsy5" is a Reserved Font Name for this Font Software.
+"cmsy6" is a Reserved Font Name for this Font Software.
+"cmsy7" is a Reserved Font Name for this Font Software.
+"cmsy8" is a Reserved Font Name for this Font Software.
+"cmsy9" is a Reserved Font Name for this Font Software.
+"cmtcsc10" is a Reserved Font Name for this Font Software.
+"cmtex10" is a Reserved Font Name for this Font Software.
+"cmtex8" is a Reserved Font Name for this Font Software.
+"cmtex9" is a Reserved Font Name for this Font Software.
+"cmti10" is a Reserved Font Name for this Font Software.
+"cmti12" is a Reserved Font Name for this Font Software.
+"cmti7" is a Reserved Font Name for this Font Software.
+"cmti8" is a Reserved Font Name for this Font Software.
+"cmti9" is a Reserved Font Name for this Font Software.
+"cmtt10" is a Reserved Font Name for this Font Software.
+"cmtt12" is a Reserved Font Name for this Font Software.
+"cmtt8" is a Reserved Font Name for this Font Software.
+"cmtt9" is a Reserved Font Name for this Font Software.
+"cmu10" is a Reserved Font Name for this Font Software.
+"cmvtt10" is a Reserved Font Name for this Font Software.
+"euex10" is a Reserved Font Name for this Font Software.
+"euex7" is a Reserved Font Name for this Font Software.
+"euex8" is a Reserved Font Name for this Font Software.
+"euex9" is a Reserved Font Name for this Font Software.
+"eufb10" is a Reserved Font Name for this Font Software.
+"eufb5" is a Reserved Font Name for this Font Software.
+"eufb7" is a Reserved Font Name for this Font Software.
+"eufm10" is a Reserved Font Name for this Font Software.
+"eufm5" is a Reserved Font Name for this Font Software.
+"eufm7" is a Reserved Font Name for this Font Software.
+"eurb10" is a Reserved Font Name for this Font Software.
+"eurb5" is a Reserved Font Name for this Font Software.
+"eurb7" is a Reserved Font Name for this Font Software.
+"eurm10" is a Reserved Font Name for this Font Software.
+"eurm5" is a Reserved Font Name for this Font Software.
+"eurm7" is a Reserved Font Name for this Font Software.
+"eusb10" is a Reserved Font Name for this Font Software.
+"eusb5" is a Reserved Font Name for this Font Software.
+"eusb7" is a Reserved Font Name for this Font Software.
+"eusm10" is a Reserved Font Name for this Font Software.
+"eusm5" is a Reserved Font Name for this Font Software.
+"eusm7" is a Reserved Font Name for this Font Software.
+"lasy10" is a Reserved Font Name for this Font Software.
+"lasy5" is a Reserved Font Name for this Font Software.
+"lasy6" is a Reserved Font Name for this Font Software.
+"lasy7" is a Reserved Font Name for this Font Software.
+"lasy8" is a Reserved Font Name for this Font Software.
+"lasy9" is a Reserved Font Name for this Font Software.
+"lasyb10" is a Reserved Font Name for this Font Software.
+"lcircle1" is a Reserved Font Name for this Font Software.
+"lcirclew" is a Reserved Font Name for this Font Software.
+"lcmss8" is a Reserved Font Name for this Font Software.
+"lcmssb8" is a Reserved Font Name for this Font Software.
+"lcmssi8" is a Reserved Font Name for this Font Software.
+"line10" is a Reserved Font Name for this Font Software.
+"linew10" is a Reserved Font Name for this Font Software.
+"msam10" is a Reserved Font Name for this Font Software.
+"msam5" is a Reserved Font Name for this Font Software.
+"msam6" is a Reserved Font Name for this Font Software.
+"msam7" is a Reserved Font Name for this Font Software.
+"msam8" is a Reserved Font Name for this Font Software.
+"msam9" is a Reserved Font Name for this Font Software.
+"msbm10" is a Reserved Font Name for this Font Software.
+"msbm5" is a Reserved Font Name for this Font Software.
+"msbm6" is a Reserved Font Name for this Font Software.
+"msbm7" is a Reserved Font Name for this Font Software.
+"msbm8" is a Reserved Font Name for this Font Software.
+"msbm9" is a Reserved Font Name for this Font Software.
+"wncyb10" is a Reserved Font Name for this Font Software.
+"wncyi10" is a Reserved Font Name for this Font Software.
+"wncyr10" is a Reserved Font Name for this Font Software.
+"wncysc10" is a Reserved Font Name for this Font Software.
+"wncyss10" is a Reserved Font Name for this Font Software.
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded, 
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.
+
+For third-party BaKoMa:
+BaKoMa Fonts Licence
+			--------------------
+
+  This licence covers two font packs (known as BaKoMa Fonts Colelction,
+  which is available at `CTAN:fonts/cm/ps-type1/bakoma/'):
+
+    1) BaKoMa-CM (1.1/12-Nov-94)
+       Computer Modern Fonts in PostScript Type 1 and TrueType font formats.
+
+    2) BaKoMa-AMS (1.2/19-Jan-95)
+       AMS TeX fonts in PostScript Type 1 and TrueType font formats.
+   
+  Copyright (C) 1994, 1995, Basil K. Malyshev. All Rights Reserved.
+
+  Permission to copy and distribute these fonts for any purpose is 
+  hereby granted without fee, provided that the above copyright notice, 
+  author statement and this permission notice appear in all copies of 
+  these fonts and related documentation.
+
+  Permission to modify and distribute modified fonts for any purpose is 
+  hereby granted without fee, provided that the copyright notice, 
+  author statement, this permission notice and location of original 
+  fonts (http://www.ctan.org/tex-archive/fonts/cm/ps-type1/bakoma)
+  appear in all copies of modified fonts and related documentation.
+
+  Permission to use these fonts (embedding into PostScript, PDF, SVG
+  and printing by using any software) is hereby granted without fee. 
+  It is not required to provide any notices about using these fonts.
+
+ Basil K. Malyshev
+ INSTITUTE FOR HIGH ENERGY PHYSICS
+ IHEP, OMVT
+ Moscow Region
+ 142281 PROTVINO
+ RUSSIA
+
+ E-Mail:	[email protected]
+      or	[email protected]
+
+For thirdparty carlogo:
+----> we renamed carlito -> carlogo to comply with the terms <----
+
+Copyright (c) 2010-2013 by tyPoland Lukasz Dziedzic with Reserved Font Name "Carlito".
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide development of collaborative font projects, to support the font creation efforts of academic and linguistic communities, and to provide a free and open framework in which fonts may be shared and improved in partnership with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and redistributed freely as long as they are not sold by themselves. The fonts, including any derivative works, can be bundled, embedded, redistributed and/or sold with any software provided that any reserved names are not used by derivative works. The fonts and derivatives, however, cannot be released under any other type of license. The requirement for fonts to remain under this license does not apply to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright Holder(s) under this license and clearly marked as such. This may include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting, or substituting -- in part or in whole -- any of the components of the Original Version, by changing formats or by porting the Font Software to a new environment.
+
+"Author" refers to any designer, engineer, programmer, technical writer or other person who contributed to the Font Software.
+
+PERMISSION & CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining a copy of the Font Software, to use, study, copy, merge, embed, modify, redistribute, and sell modified and unmodified copies of the Font Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components, in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled, redistributed and/or sold with any software, provided that each copy contains the above copyright notice and this license. These can be included either as stand-alone text files, human-readable headers or in the appropriate machine-readable metadata fields within text or binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font Name(s) unless explicit written permission is granted by the corresponding Copyright Holder. This restriction only applies to the primary font name as presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font Software shall not be used to promote, endorse or advertise any Modified Version, except to acknowledge the contribution(s) of the Copyright Holder(s) and the Author(s) or with their explicit written permission.
+
+5) The Font Software, modified or unmodified, in part or in whole, must be distributed entirely under this license, and must not be distributed under any other license. The requirement for fonts to remain under this license does not apply to any document created using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE.
+
+For third-party ColorBrewer Color Schemes:
+Apache-Style Software License for ColorBrewer Color Schemes
+
+Version 1.1
+
+Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania 
+State University. All rights reserved. Redistribution and use in source 
+and binary forms, with or without modification, are permitted provided 
+that the following conditions are met:
+
+1. Redistributions as source code must retain the above copyright notice, 
+this list of conditions and the following disclaimer.
+
+2. The end-user documentation included with the redistribution, if any, 
+must include the following acknowledgment: "This product includes color 
+specifications and designs developed by Cynthia Brewer 
+(http://colorbrewer.org/)." Alternately, this acknowledgment may appear in 
+the software itself, if and wherever such third-party acknowledgments 
+normally appear.
+
+3. The name "ColorBrewer" must not be used to endorse or promote products 
+derived from this software without prior written permission. For written 
+permission, please contact Cynthia Brewer at [email protected].
+
+4. Products derived from this software may not be called "ColorBrewer", 
+nor may "ColorBrewer" appear in their name, without prior written 
+permission of Cynthia Brewer.
+
+THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, 
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 
+AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
+CYNTHIA BREWER, MARK HARROWER, OR THE PENNSYLVANIA STATE UNIVERSITY BE 
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.
+
+For third-party JSXTOOLS_RESIZE_OBSERVER:
+# CC0 1.0 Universal
+
+## Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an “owner”) of an original work of
+authorship and/or a database (each, a “Work”).
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific works
+(“Commons”) that the public can reliably and without fear of later claims of
+infringement build upon, modify, incorporate in other works, reuse and
+redistribute as freely as possible in any form whatsoever and for any purposes,
+including without limitation commercial purposes. These owners may contribute
+to the Commons to promote the ideal of a free culture and the further
+production of creative, cultural and scientific works, or to gain reputation or
+greater distribution for their Work in part through the use and efforts of
+others.
+
+For these and/or other purposes and motivations, and without any expectation of
+additional consideration or compensation, the person associating CC0 with a
+Work (the “Affirmer”), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and
+publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+   protected by copyright and related or neighboring rights (“Copyright and
+   Related Rights”). Copyright and Related Rights include, but are not limited
+   to, the following:
+   1. the right to reproduce, adapt, distribute, perform, display, communicate,
+      and translate a Work;
+   2. moral rights retained by the original author(s) and/or performer(s);
+   3. publicity and privacy rights pertaining to a person’s image or likeness
+      depicted in a Work;
+   4. rights protecting against unfair competition in regards to a Work,
+      subject to the limitations in paragraph 4(i), below;
+   5. rights protecting the extraction, dissemination, use and reuse of data in
+      a Work;
+   6. database rights (such as those arising under Directive 96/9/EC of the
+      European Parliament and of the Council of 11 March 1996 on the legal
+      protection of databases, and under any national implementation thereof,
+      including any amended or successor version of such directive); and
+   7. other similar, equivalent or corresponding rights throughout the world
+      based on applicable law or treaty, and any national implementations
+      thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of,
+   applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+   unconditionally waives, abandons, and surrenders all of Affirmer’s Copyright
+   and Related Rights and associated claims and causes of action, whether now
+   known or unknown (including existing as well as future claims and causes of
+   action), in the Work (i) in all territories worldwide, (ii) for the maximum
+   duration provided by applicable law or treaty (including future time
+   extensions), (iii) in any current or future medium and for any number of
+   copies, and (iv) for any purpose whatsoever, including without limitation
+   commercial, advertising or promotional purposes (the “Waiver”). Affirmer
+   makes the Waiver for the benefit of each member of the public at large and
+   to the detriment of Affirmer’s heirs and successors, fully intending that
+   such Waiver shall not be subject to revocation, rescission, cancellation,
+   termination, or any other legal or equitable action to disrupt the quiet
+   enjoyment of the Work by the public as contemplated by Affirmer’s express
+   Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be
+   judged legally invalid or ineffective under applicable law, then the Waiver
+   shall be preserved to the maximum extent permitted taking into account
+   Affirmer’s express Statement of Purpose. In addition, to the extent the
+   Waiver is so judged Affirmer hereby grants to each affected person a
+   royalty-free, non transferable, non sublicensable, non exclusive,
+   irrevocable and unconditional license to exercise Affirmer’s Copyright and
+   Related Rights in the Work (i) in all territories worldwide, (ii) for the
+   maximum duration provided by applicable law or treaty (including future time
+   extensions), (iii) in any current or future medium and for any number of
+   copies, and (iv) for any purpose whatsoever, including without limitation
+   commercial, advertising or promotional purposes (the “License”). The License
+   shall be deemed effective as of the date CC0 was applied by Affirmer to the
+   Work. Should any part of the License for any reason be judged legally
+   invalid or ineffective under applicable law, such partial invalidity or
+   ineffectiveness shall not invalidate the remainder of the License, and in
+   such case Affirmer hereby affirms that he or she will not (i) exercise any
+   of his or her remaining Copyright and Related Rights in the Work or (ii)
+   assert any associated claims and causes of action with respect to the Work,
+   in either case contrary to Affirmer’s express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+   1. No trademark or patent rights held by Affirmer are waived, abandoned,
+      surrendered, licensed or otherwise affected by this document.
+   2. Affirmer offers the Work as-is and makes no representations or warranties
+      of any kind concerning the Work, express, implied, statutory or
+      otherwise, including without limitation warranties of title,
+      merchantability, fitness for a particular purpose, non infringement, or
+      the absence of latent or other defects, accuracy, or the present or
+      absence of errors, whether or not discoverable, all to the greatest
+      extent permissible under applicable law.
+   3. Affirmer disclaims responsibility for clearing rights of other persons
+      that may apply to the Work or any use thereof, including without
+      limitation any person’s Copyright and Related Rights in the Work.
+      Further, Affirmer disclaims responsibility for obtaining any necessary
+      consents, permissions or other rights required for any use of the Work.
+   4. Affirmer understands and acknowledges that Creative Commons is not a
+      party to this document and has no duty or obligation with respect to this
+      CC0 or use of the Work.
+
+For more information, please see
+http://creativecommons.org/publicdomain/zero/1.0/.
+
+For third-party QT4_EDITOR:
+Module creating PyQt4 form dialogs/layouts to edit various type of parameters
+
+
+formlayout License Agreement (MIT License)
+------------------------------------------
+
+Copyright (c) 2009 Pierre Raybaut
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+For third-party SOLARIZED:
+https://github.com/altercation/solarized/blob/master/LICENSE
+Copyright (c) 2011 Ethan Schoonover
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+For third-party STIX:
+TERMS AND CONDITIONS
+
+   1. Permission is hereby granted, free of charge, to any person 
+obtaining a copy of the STIX Fonts-TM set accompanying this license 
+(collectively, the "Fonts") and the associated documentation files 
+(collectively with the Fonts, the "Font Software"), to reproduce and 
+distribute the Font Software, including the rights to use, copy, merge 
+and publish copies of the Font Software, and to permit persons to whom 
+the Font Software is furnished to do so same, subject to the following 
+terms and conditions (the "License").
+
+   2. The following copyright and trademark notice and these Terms and 
+Conditions shall be included in all copies of one or more of the Font 
+typefaces and any derivative work created as permitted under this 
+License:
+
+	Copyright (c) 2001-2005 by the STI Pub Companies, consisting of 
+the American Institute of Physics, the American Chemical Society, the 
+American Mathematical Society, the American Physical Society, Elsevier, 
+Inc., and The Institute of Electrical and Electronic Engineers, Inc. 
+Portions copyright (c) 1998-2003 by MicroPress, Inc. Portions copyright 
+(c) 1990 by Elsevier, Inc. All rights reserved. STIX Fonts-TM is a 
+trademark of The Institute of Electrical and Electronics Engineers, Inc.
+
+   3. You may (a) convert the Fonts from one format to another (e.g., 
+from TrueType to PostScript), in which case the normal and reasonable 
+distortion that occurs during such conversion shall be permitted and (b) 
+embed or include a subset of the Fonts in a document for the purposes of 
+allowing users to read text in the document that utilizes the Fonts. In 
+each case, you may use the STIX Fonts-TM mark to designate the resulting 
+Fonts or subset of the Fonts.
+
+   4. You may also (a) add glyphs or characters to the Fonts, or modify 
+the shape of existing glyphs, so long as the base set of glyphs is not 
+removed and (b) delete glyphs or characters from the Fonts, provided 
+that the resulting font set is distributed with the following 
+disclaimer: "This [name] font does not include all the Unicode points 
+covered in the STIX Fonts-TM set but may include others." In each case, 
+the name used to denote the resulting font set shall not include the 
+term "STIX" or any similar term.
+
+   5. You may charge a fee in connection with the distribution of the 
+Font Software, provided that no copy of one or more of the individual 
+Font typefaces that form the STIX Fonts-TM set may be sold by itself.
+
+   6. THE FONT SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY 
+KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES 
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 
+OF COPYRIGHT, PATENT, TRADEMARK OR OTHER RIGHT. IN NO EVENT SHALL 
+MICROPRESS OR ANY OF THE STI PUB COMPANIES BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, INCLUDING, BUT NOT LIMITED TO, ANY GENERAL, 
+SPECIAL, INDIRECT, INCIDENTAL OR CONSEQUENTIAL DAMAGES, WHETHER IN AN 
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM OR OUT OF THE USE OR 
+INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT 
+SOFTWARE.
+
+   7. Except as contained in the notice set forth in Section 2, the 
+names MicroPress Inc. and STI Pub Companies, as well as the names of the 
+companies/organizations that compose the STI Pub Companies, shall not be 
+used in advertising or otherwise to promote the sale, use or other 
+dealings in the Font Software without the prior written consent of the 
+respective company or organization.
+
+   8. This License shall become null and void in the event of any 
+material breach of the Terms and Conditions herein by licensee.
+
+   9. A substantial portion of the STIX Fonts set was developed by 
+MicroPress Inc. for the STI Pub Companies. To obtain additional 
+mathematical fonts, please contact MicroPress, Inc., 68-30 Harrow 
+Street, Forest Hills, NY 11375, USA - Phone: (718) 575-1816.
+
+For third-party YORICK:
+BSD-style license for gist/yorick colormaps.
+
+Copyright:
+
+  Copyright (c) 1996.  The Regents of the University of California.
+			 All rights reserved.
+
+Permission to use, copy, modify, and distribute this software for any
+purpose without fee is hereby granted, provided that this entire
+notice is included in all copies of any software which is or includes
+a copy or modification of this software and in all copies of the
+supporting documentation for such software.
+
+This work was produced at the University of California, Lawrence
+Livermore National Laboratory under contract no. W-7405-ENG-48 between
+the U.S. Department of Energy and The Regents of the University of
+California for the operation of UC LLNL.
+
+
+			      DISCLAIMER
+
+This software was prepared as an account of work sponsored by an
+agency of the United States Government.  Neither the United States
+Government nor the University of California nor any of their
+employees, makes any warranty, express or implied, or assumes any
+liability or responsibility for the accuracy, completeness, or
+usefulness of any information, apparatus, product, or process
+disclosed, or represents that its use would not infringe
+privately-owned rights.  Reference herein to any specific commercial
+products, process, or service by trade name, trademark, manufacturer,
+or otherwise, does not necessarily constitute or imply its
+endorsement, recommendation, or favoring by the United States
+Government or the University of California.  The views and opinions of
+authors expressed herein do not necessarily state or reflect those of
+the United States Government or the University of California, and
+shall not be used for advertising or product endorsement purposes.
+
+
+				AUTHOR
+
+David H. Munro wrote Yorick and Gist.  Berkeley Yacc (byacc) generated
+the Yorick parser.  The routines in Math are from LAPACK and FFTPACK;
+MathC contains C translations by David H. Munro.  The algorithms for
+Yorick's random number generator and several special functions in
+Yorick/include were taken from Numerical Recipes by Press, et. al.,
+although the Yorick implementations are unrelated to those in
+Numerical Recipes.  A small amount of code in Gist was adapted from
+the X11R4 release, copyright M.I.T. -- the complete copyright notice
+may be found in the (unused) file Gist/host.c.
+
+
+
+Open Source Software Licensed under the PSF License Version 2:
+--------------------------------------------------------------------
+1. Python
+Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation. All rights reserved.
+
+Copyright (c) 2000 BeOpen.com. All rights reserved.
+
+Copyright (c) 1995-2001 Corporation for National Research Initiatives. All rights reserved.
+
+Copyright (c) 1991-1995 Stichting Mathematisch Centrum. All rights reserved.
+
+
+Terms of the PSF License Version 2:
+--------------------------------------------------------------------
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
+Rights Reserved" are retained in Python alone or in any derivative version
+prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+
+Open Source Software Licensed under the MIT License:
+The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) 2021 THL A29 Limited.
+--------------------------------------------------------------------
+1. C^3 Framework
+Copyright (c) 2018 Junyu Gao
+
+
+Terms of the MIT License:
+--------------------------------------------------------------------
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
+Open Source Software Licensed under the MIT License:
+--------------------------------------------------------------------
+1. tensorboardX
+Copyright (c) 2017 Tzu-Wei Huang
+
+
+A copy of the MIT License is included in this file.
+
+
+
+Open Source Software Licensed under the Apache License Version 2.0:
+The below software in this distribution may have been modified by Tencent.
+--------------------------------------------------------------------
+1. DETR
+Copyright 2020 - present, Facebook, Inc
+Please note this software has been modified by Tencent in this distribution.
+
+
+Terms of the Apache License Version 2.0:
+--------------------------------------------------------------------
+Apache License 
+
+Version 2.0, January 2004
+
+http://www.apache.org/licenses/ 
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of this License; and 
+
+You must cause any modified files to carry prominent notices stating that You changed the files; and 
+
+You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 
+
+If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 
+
+You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS

crowd_counter/README.md

@@ -0,0 +1,164 @@
+# P2PNet (ICCV2021 Oral Presentation)
+
+This repository contains codes for the official implementation in PyTorch of **P2PNet** as described in [Rethinking Counting and Localization in Crowds: A Purely Point-Based Framework](https://arxiv.org/abs/2107.12746).
+ 
+A brief introduction of P2PNet can be found at [机器之心 (almosthuman)](https://mp.weixin.qq.com/s?__biz=MzA3MzI4MjgzMw==&mid=2650827826&idx=3&sn=edd3d66444130fb34a59d08fab618a9e&chksm=84e5a84cb392215a005a3b3424f20a9d24dc525dcd933960035bf4b6aa740191b5ecb2b7b161&mpshare=1&scene=1&srcid=1004YEOC7HC9daYRYeUio7Xn&sharer_sharetime=1633675738338&sharer_shareid=7d375dccd3b2f9eec5f8b27ee7c04883&version=3.1.16.5505&platform=win#rd).
+
+The codes is tested with PyTorch 1.5.0. It may not run with other versions.
+
+## Visualized demos for P2PNet
+<img src="vis/congested1.png" width="1000"/>   
+<img src="vis/congested2.png" width="1000"/> 
+<img src="vis/congested3.png" width="1000"/> 
+
+## The network
+The overall architecture of the P2PNet. Built upon the VGG16, it firstly introduce an upsampling path to obtain fine-grained feature map. 
+Then it exploits two branches to simultaneously predict a set of point proposals and their confidence scores.
+
+<img src="vis/net.png" width="1000"/>   
+
+## Comparison with state-of-the-art methods
+The P2PNet achieved state-of-the-art performance on several challenging datasets with various densities.
+
+| Methods   | Venue     | SHTechPartA <br> MAE/MSE  |SHTechPartB <br> MAE/MSE | UCF_CC_50 <br> MAE/MSE | UCF_QNRF <br> MAE/MSE   |
+|:----:|:----:|:----:|:----:|:----:|:----:|
+CAN  | CVPR'19 | 62.3/100.0 | 7.8/12.2 | 212.2/**243.7** | 107.0/183.0 |
+Bayesian+ | ICCV'19 | 62.8/101.8 | 7.7/12.7 | 229.3/308.2 | 88.7/154.8 |
+S-DCNet  | ICCV'19 | 58.3/95.0 | 6.7/10.7 | 204.2/301.3 | 104.4/176.1 |
+SANet+SPANet  | ICCV'19 | 59.4/92.5 | 6.5/**9.9** | 232.6/311.7 | -/- |
+DUBNet  | AAAI'20 | 64.6/106.8 | 7.7/12.5 | 243.8/329.3 | 105.6/180.5 |
+SDANet | AAAI'20 | 63.6/101.8 | 7.8/10.2 | 227.6/316.4 | -/- |
+ADSCNet | CVPR'20 | <u>55.4</u>/97.7 | <u>6.4</u>/11.3 | 198.4/267.3 | **71.3**/**132.5**|
+ASNet   | CVPR'20 | 57.78/<u>90.13</u> | -/- | <u>174.84</u>/<u>251.63</u> | 91.59/159.71 |
+AMRNet  | ECCV'20 | 61.59/98.36 | 7.02/11.00 | 184.0/265.8 | 86.6/152.2 |
+AMSNet  | ECCV'20 | 56.7/93.4 | 6.7/10.2 | 208.4/297.3 | 101.8/163.2|
+DM-Count  | NeurIPS'20 | 59.7/95.7 | 7.4/11.8 | 211.0/291.5 | 85.6/<u>148.3</u>|
+**Ours** |- | **52.74**/**85.06** | **6.25**/**9.9** | **172.72**/256.18 | <u>85.32</u>/154.5 |
+
+Comparison on the [NWPU-Crowd](https://www.crowdbenchmark.com/resultdetail.html?rid=81) dataset.
+
+| Methods   | MAE[O]  |MSE[O] | MAE[L] | MAE[S]   |
+|:----:|:----:|:----:|:----:|:----:|
+MCNN  | 232.5|714.6 | 220.9|1171.9 |
+SANet  | 190.6 | 491.4 | 153.8 | 716.3|
+CSRNet | 121.3 | 387.8 | 112.0 | <u>522.7</u> |
+PCC-Net  | 112.3 | 457.0 | 111.0 | 777.6 |
+CANNet  | 110.0 | 495.3 | 102.3 | 718.3|
+Bayesian+  | 105.4 | 454.2 | 115.8 | 750.5 |
+S-DCNet   | 90.2 | 370.5 | **82.9** | 567.8 |
+DM-Count  | <u>88.4</u> | 388.6 | 88.0 | **498.0** |
+**Ours** | **77.44**|**362** | <u>83.28</u>| 553.92 |
+
+The overall performance for both counting and localization.
+
+|nAP$_{\delta}$|SHTechPartA| SHTechPartB | UCF_CC_50 | UCF_QNRF | NWPU_Crowd |
+|:----:|:----:|:----:|:----:|:----:|:----:|    
+$\delta=0.05$ | 10.9\% | 23.8\%  | 5.0\% | 5.9\% | 12.9\% | 
+$\delta=0.25$ | 70.3\% | 84.2\%  | 54.5\% | 55.4\% | 71.3\% |  
+$\delta=0.50$ | 90.1\% | 94.1\%  | 88.1\% | 83.2\% | 89.1\% | 
+$\delta=\{{0.05:0.05:0.50}\}$ | 64.4\% | 76.3\%  | 54.3\% | 53.1\% | 65.0\% |  
+
+Comparison for the localization performance in terms of F1-Measure on NWPU.
+
+| Method| F1-Measure |Precision| Recall |
+|:----:|:----:|:----:|:----:|
+FasterRCNN  |  0.068 |  0.958 | 0.035 |
+TinyFaces |  0.567  |  0.529 | 0.611 |
+RAZ |   0.599 |  0.666 |  0.543|
+Crowd-SDNet |  0.637  | 0.651  | 0.624  |
+PDRNet |  0.653 | 0.675  | 0.633  |
+TopoCount | 0.692  | 0.683  | **0.701** |
+D2CNet | <u>0.700</u> | **0.741**  | 0.662 |
+**Ours** |**0.712** | <u>0.729</u>  | <u>0.695</u> |
+
+## Installation
+* Clone this repo into a directory named P2PNET_ROOT
+* Organize your datasets as required
+* Install Python dependencies. We use python 3.6.5 and pytorch 1.5.0
+```
+pip install -r requirements.txt
+```
+
+## Organize the counting dataset
+We use a list file to collect all the images and their ground truth annotations in a counting dataset. When your dataset is organized as recommended in the following, the format of this list file is defined as:
+```
+train/scene01/img01.jpg train/scene01/img01.txt
+train/scene01/img02.jpg train/scene01/img02.txt
+...
+train/scene02/img01.jpg train/scene02/img01.txt
+```
+
+### Dataset structures:
+```
+DATA_ROOT/
+        |->train/
+        |    |->scene01/
+        |    |->scene02/
+        |    |->...
+        |->test/
+        |    |->scene01/
+        |    |->scene02/
+        |    |->...
+        |->train.list
+        |->test.list
+```
+DATA_ROOT is your path containing the counting datasets.
+
+### Annotations format
+For the annotations of each image, we use a single txt file which contains one annotation per line. Note that indexing for pixel values starts at 0. The expected format of each line is:
+```
+x1 y1
+x2 y2
+...
+```
+
+## Training
+
+The network can be trained using the `train.py` script. For training on SHTechPartA, use
+
+```
+CUDA_VISIBLE_DEVICES=0 python train.py --data_root $DATA_ROOT \
+    --dataset_file SHHA \
+    --epochs 3500 \
+    --lr_drop 3500 \
+    --output_dir ./logs \
+    --checkpoints_dir ./weights \
+    --tensorboard_dir ./logs \
+    --lr 0.0001 \
+    --lr_backbone 0.00001 \
+    --batch_size 8 \
+    --eval_freq 1 \
+    --gpu_id 0
+```
+By default, a periodic evaluation will be conducted on the validation set.
+
+## Testing
+
+A trained model (with an MAE of **51.96**) on SHTechPartA is available at "./weights", run the following commands to launch a visualization demo:
+
+```
+CUDA_VISIBLE_DEVICES=0 python run_test.py --weight_path ./weights/SHTechA.pth --output_dir ./logs/
+```
+
+## Acknowledgements
+
+- Part of codes are borrowed from the [C^3 Framework](https://github.com/gjy3035/C-3-Framework).
+- We refer to [DETR](https://github.com/facebookresearch/detr) to implement our matching strategy.
+
+
+## Citing P2PNet
+
+If you find P2PNet is useful in your project, please consider citing us:
+
+```BibTeX
+@inproceedings{song2021rethinking,
+  title={Rethinking Counting and Localization in Crowds: A Purely Point-Based Framework},
+  author={Song, Qingyu and Wang, Changan and Jiang, Zhengkai and Wang, Yabiao and Tai, Ying and Wang, Chengjie and Li, Jilin and Huang, Feiyue and Wu, Yang},
+  journal={Proceedings of the IEEE/CVF International Conference on Computer Vision},
+  year={2021}
+}
+```
+
+## Related works from Tencent Youtu Lab
+- [AAAI2021] To Choose or to Fuse? Scale Selection for Crowd Counting. ([paper link](https://ojs.aaai.org/index.php/AAAI/article/view/16360) & [codes](https://github.com/TencentYoutuResearch/CrowdCounting-SASNet))
+- [ICCV2021] Uniformity in Heterogeneity: Diving Deep into Count Interval Partition for Crowd Counting. ([paper link](https://arxiv.org/abs/2107.12619) & [codes](https://github.com/TencentYoutuResearch/CrowdCounting-UEPNet))

crowd_counter/__init__.py

@@ -0,0 +1,117 @@
+import torch
+import torchvision.transforms as standard_transforms
+import numpy as np
+
+from PIL import Image
+import cv2
+
+from .engine import *
+from .models import build_model
+import os
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class Args:
+    def __init__(
+        self,
+        backbone: str,
+        row: int,
+        line: int,
+        output_dir: str,
+        weight_path: str,
+        # gpu_id: int,
+    ) -> None:
+        self.backbone = backbone
+        self.row = row
+        self.line = line
+        self.output_dir = output_dir
+        self.weight_path = weight_path
+        # self.gpu_id = gpu_id
+
+
+class CrowdCounter:
+    def __init__(self) -> None:
+        # Create the Args object
+        self.args = Args(
+            backbone="vgg16_bn",
+            row=2,
+            line=2,
+            output_dir="./crowd_counter/preds",
+            weight_path="./crowd_counter/weights/SHTechA.pth",
+            # gpu_id=0,
+        )
+
+        # device = torch.device('cuda')
+        self.device = torch.device("cpu")
+        # get the P2PNet
+        self.model = build_model(self.args)
+        # move to GPU
+        self.model.to(self.device)
+        # load trained model
+        if self.args.weight_path is not None:
+            checkpoint = torch.load(self.args.weight_path, map_location="cpu")
+            self.model.load_state_dict(checkpoint["model"])
+        # convert to eval mode
+        self.model.eval()
+        # create the pre-processing transform
+        self.transform = standard_transforms.Compose(
+            [
+                standard_transforms.ToTensor(),
+                standard_transforms.Normalize(
+                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
+                ),
+            ]
+        )
+
+    def test(
+        self, args: Args, img_raw: Image.Image , debug: bool = False,
+    ) -> tuple[any, Image.Image, torch.Tensor]:
+        
+        # round the size
+        width, height = img_raw.size
+        new_width = width // 128 * 128
+        new_height = height // 128 * 128
+        img_raw = img_raw.resize((new_width, new_height), Image.LANCZOS)
+        # pre-proccessing
+        img = self.transform(img_raw)
+
+        samples = torch.Tensor(img).unsqueeze(0)
+        samples = samples.to(self.device)
+        # run inference
+        outputs = self.model(samples)
+        outputs_scores = torch.nn.functional.softmax(outputs["pred_logits"], -1)[
+            :, :, 1
+        ][0]
+
+        outputs_points = outputs["pred_points"][0]
+
+        threshold = 0.5
+        # filter the predictions
+        conf = outputs_scores[outputs_scores > threshold]
+        points = (
+            outputs_points[outputs_scores > threshold].detach().cpu().numpy().tolist()
+        )
+
+        # draw the predictions
+        size = 5
+        img_to_draw = cv2.cvtColor(np.array(img_raw), cv2.COLOR_RGB2BGR)
+
+        for p in points:
+            img_to_draw = cv2.circle(
+                img_to_draw, (int(p[0]), int(p[1])), size, (255, 0, 0), -1
+            )
+        return points, img_to_draw, conf
+
+    # Function to process and save images
+    def inference(self, img_raw: Image.Image) -> tuple[int, Image.Image]:
+
+        # Predict points on the image
+        points, img_to_draw, conf = self.test(self.args, img_raw)
+
+        # Prepare text for the number of points
+        num_points = len(points)
+
+        # Pilgrims, Drawn Image %
+        return num_points, img_to_draw

crowd_counter/crowd_datasets/SHHA/SHHA.py

@@ -0,0 +1,133 @@
+import os
+import random
+import torch
+import numpy as np
+from torch.utils.data import Dataset
+from PIL import Image
+import cv2
+import glob
+import scipy.io as io
+
+class SHHA(Dataset):
+    def __init__(self, data_root, transform=None, train=False, patch=False, flip=False):
+        self.root_path = data_root
+        self.train_lists = "shanghai_tech_part_a_train.list"
+        self.eval_list = "shanghai_tech_part_a_test.list"
+        # there may exist multiple list files
+        self.img_list_file = self.train_lists.split(',')
+        if train:
+            self.img_list_file = self.train_lists.split(',')
+        else:
+            self.img_list_file = self.eval_list.split(',')
+
+        self.img_map = {}
+        self.img_list = []
+        # loads the image/gt pairs
+        for _, train_list in enumerate(self.img_list_file):
+            train_list = train_list.strip()
+            with open(os.path.join(self.root_path, train_list)) as fin:
+                for line in fin:
+                    if len(line) < 2: 
+                        continue
+                    line = line.strip().split()
+                    self.img_map[os.path.join(self.root_path, line[0].strip())] = \
+                                    os.path.join(self.root_path, line[1].strip())
+        self.img_list = sorted(list(self.img_map.keys()))
+        # number of samples
+        self.nSamples = len(self.img_list)
+        
+        self.transform = transform
+        self.train = train
+        self.patch = patch
+        self.flip = flip
+
+    def __len__(self):
+        return self.nSamples
+
+    def __getitem__(self, index):
+        assert index <= len(self), 'index range error'
+
+        img_path = self.img_list[index]
+        gt_path = self.img_map[img_path]
+        # load image and ground truth
+        img, point = load_data((img_path, gt_path), self.train)
+        # applu augumentation
+        if self.transform is not None:
+            img = self.transform(img)
+
+        if self.train:
+            # data augmentation -> random scale
+            scale_range = [0.7, 1.3]
+            min_size = min(img.shape[1:])
+            scale = random.uniform(*scale_range)
+            # scale the image and points
+            if scale * min_size > 128:
+                img = torch.nn.functional.upsample_bilinear(img.unsqueeze(0), scale_factor=scale).squeeze(0)
+                point *= scale
+        # random crop augumentaiton
+        if self.train and self.patch:
+            img, point = random_crop(img, point)
+            for i, _ in enumerate(point):
+                point[i] = torch.Tensor(point[i])
+        # random flipping
+        if random.random() > 0.5 and self.train and self.flip:
+            # random flip
+            img = torch.Tensor(img[:, :, :, ::-1].copy())
+            for i, _ in enumerate(point):
+                point[i][:, 0] = 128 - point[i][:, 0]
+
+        if not self.train:
+            point = [point]
+
+        img = torch.Tensor(img)
+        # pack up related infos
+        target = [{} for i in range(len(point))]
+        for i, _ in enumerate(point):
+            target[i]['point'] = torch.Tensor(point[i])
+            image_id = int(img_path.split('/')[-1].split('.')[0].split('_')[-1])
+            image_id = torch.Tensor([image_id]).long()
+            target[i]['image_id'] = image_id
+            target[i]['labels'] = torch.ones([point[i].shape[0]]).long()
+
+        return img, target
+
+
+def load_data(img_gt_path, train):
+    img_path, gt_path = img_gt_path
+    # load the images
+    img = cv2.imread(img_path)
+    img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
+    # load ground truth points
+    points = []
+    with open(gt_path) as f_label:
+        for line in f_label:
+            x = float(line.strip().split(' ')[0])
+            y = float(line.strip().split(' ')[1])
+            points.append([x, y])
+
+    return img, np.array(points)
+
+# random crop augumentation
+def random_crop(img, den, num_patch=4):
+    half_h = 128
+    half_w = 128
+    result_img = np.zeros([num_patch, img.shape[0], half_h, half_w])
+    result_den = []
+    # crop num_patch for each image
+    for i in range(num_patch):
+        start_h = random.randint(0, img.size(1) - half_h)
+        start_w = random.randint(0, img.size(2) - half_w)
+        end_h = start_h + half_h
+        end_w = start_w + half_w
+        # copy the cropped rect
+        result_img[i] = img[:, start_h:end_h, start_w:end_w]
+        # copy the cropped points
+        idx = (den[:, 0] >= start_w) & (den[:, 0] <= end_w) & (den[:, 1] >= start_h) & (den[:, 1] <= end_h)
+        # shift the corrdinates
+        record_den = den[idx]
+        record_den[:, 0] -= start_w
+        record_den[:, 1] -= start_h
+
+        result_den.append(record_den)
+
+    return result_img, result_den

crowd_counter/crowd_datasets/SHHA/loading_data.py

@@ -0,0 +1,27 @@
+import torchvision.transforms as standard_transforms
+from .SHHA import SHHA
+
+# DeNormalize used to get original images
+class DeNormalize(object):
+    def __init__(self, mean, std):
+        self.mean = mean
+        self.std = std
+
+    def __call__(self, tensor):
+        for t, m, s in zip(tensor, self.mean, self.std):
+            t.mul_(s).add_(m)
+        return tensor
+
+def loading_data(data_root):
+    # the pre-proccssing transform
+    transform = standard_transforms.Compose([
+        standard_transforms.ToTensor(), 
+        standard_transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                    std=[0.229, 0.224, 0.225]),
+    ])
+    # create the training dataset
+    train_set = SHHA(data_root, train=True, transform=transform, patch=True, flip=True)
+    # create the validation dataset
+    val_set = SHHA(data_root, train=False, transform=transform)
+
+    return train_set, val_set

crowd_counter/crowd_datasets/__init__.py

@@ -0,0 +1,7 @@
+# build dataset according to given 'dataset_file'
+def build_dataset(args):
+    if args.dataset_file == 'SHHA':
+        from crowd_datasets.SHHA.loading_data import loading_data
+        return loading_data
+
+    return None

crowd_counter/engine.py

@@ -0,0 +1,159 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+Train and eval functions used in main.py
+Mostly copy-paste from DETR (https://github.com/facebookresearch/detr).
+"""
+import math
+import os
+import sys
+from typing import Iterable
+
+import torch
+
+import crowd_counter.util.misc as utils
+from crowd_counter.util.misc import NestedTensor
+import numpy as np
+import time
+import torchvision.transforms as standard_transforms
+import cv2
+
+class DeNormalize(object):
+    def __init__(self, mean, std):
+        self.mean = mean
+        self.std = std
+
+    def __call__(self, tensor):
+        for t, m, s in zip(tensor, self.mean, self.std):
+            t.mul_(s).add_(m)
+        return tensor
+
+def vis(samples, targets, pred, vis_dir, des=None):
+    '''
+    samples -> tensor: [batch, 3, H, W]
+    targets -> list of dict: [{'points':[], 'image_id': str}]
+    pred -> list: [num_preds, 2]
+    '''
+    gts = [t['point'].tolist() for t in targets]
+
+    pil_to_tensor = standard_transforms.ToTensor()
+
+    restore_transform = standard_transforms.Compose([
+        DeNormalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        standard_transforms.ToPILImage()
+    ])
+    # draw one by one
+    for idx in range(samples.shape[0]):
+        sample = restore_transform(samples[idx])
+        sample = pil_to_tensor(sample.convert('RGB')).numpy() * 255
+        sample_gt = sample.transpose([1, 2, 0])[:, :, ::-1].astype(np.uint8).copy()
+        sample_pred = sample.transpose([1, 2, 0])[:, :, ::-1].astype(np.uint8).copy()
+
+        max_len = np.max(sample_gt.shape)
+
+        size = 2
+        # draw gt
+        for t in gts[idx]:
+            sample_gt = cv2.circle(sample_gt, (int(t[0]), int(t[1])), size, (0, 255, 0), -1)
+        # draw predictions
+        for p in pred[idx]:
+            sample_pred = cv2.circle(sample_pred, (int(p[0]), int(p[1])), size, (0, 0, 255), -1)
+
+        name = targets[idx]['image_id']
+        # save the visualized images
+        if des is not None:
+            cv2.imwrite(os.path.join(vis_dir, '{}_{}_gt_{}_pred_{}_gt.jpg'.format(int(name), 
+                                                des, len(gts[idx]), len(pred[idx]))), sample_gt)
+            cv2.imwrite(os.path.join(vis_dir, '{}_{}_gt_{}_pred_{}_pred.jpg'.format(int(name), 
+                                                des, len(gts[idx]), len(pred[idx]))), sample_pred)
+        else:
+            cv2.imwrite(
+                os.path.join(vis_dir, '{}_gt_{}_pred_{}_gt.jpg'.format(int(name), len(gts[idx]), len(pred[idx]))),
+                sample_gt)
+            cv2.imwrite(
+                os.path.join(vis_dir, '{}_gt_{}_pred_{}_pred.jpg'.format(int(name), len(gts[idx]), len(pred[idx]))),
+                sample_pred)
+
+# the training routine
+def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module,
+                    data_loader: Iterable, optimizer: torch.optim.Optimizer,
+                    device: torch.device, epoch: int, max_norm: float = 0):
+    model.train()
+    criterion.train()
+    metric_logger = utils.MetricLogger(delimiter="  ")
+    metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))
+    # iterate all training samples
+    for samples, targets in data_loader:
+        samples = samples.to(device)
+        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
+        # forward
+        outputs = model(samples)
+        # calc the losses
+        loss_dict = criterion(outputs, targets)
+        weight_dict = criterion.weight_dict
+        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)
+
+        # reduce all losses
+        loss_dict_reduced = utils.reduce_dict(loss_dict)
+        loss_dict_reduced_unscaled = {f'{k}_unscaled': v
+                                      for k, v in loss_dict_reduced.items()}
+        loss_dict_reduced_scaled = {k: v * weight_dict[k]
+                                    for k, v in loss_dict_reduced.items() if k in weight_dict}
+        losses_reduced_scaled = sum(loss_dict_reduced_scaled.values())
+
+        loss_value = losses_reduced_scaled.item()
+
+        if not math.isfinite(loss_value):
+            print("Loss is {}, stopping training".format(loss_value))
+            print(loss_dict_reduced)
+            sys.exit(1)
+        # backward
+        optimizer.zero_grad()
+        losses.backward()
+        if max_norm > 0:
+            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm)
+        optimizer.step()
+        # update logger
+        metric_logger.update(loss=loss_value, **loss_dict_reduced_scaled, **loss_dict_reduced_unscaled)
+        metric_logger.update(lr=optimizer.param_groups[0]["lr"])
+    # gather the stats from all processes
+    metric_logger.synchronize_between_processes()
+    print("Averaged stats:", metric_logger)
+    return {k: meter.global_avg for k, meter in metric_logger.meters.items()}
+
+# the inference routine
+@torch.no_grad()
+def evaluate_crowd_no_overlap(model, data_loader, device, vis_dir=None):
+    model.eval()
+
+    metric_logger = utils.MetricLogger(delimiter="  ")
+    metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}'))
+    # run inference on all images to calc MAE
+    maes = []
+    mses = []
+    for samples, targets in data_loader:
+        samples = samples.to(device)
+
+        outputs = model(samples)
+        outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]
+
+        outputs_points = outputs['pred_points'][0]
+
+        gt_cnt = targets[0]['point'].shape[0]
+        # 0.5 is used by default
+        threshold = 0.5
+
+        points = outputs_points[outputs_scores > threshold].detach().cpu().numpy().tolist()
+        predict_cnt = int((outputs_scores > threshold).sum())
+        # if specified, save the visualized images
+        if vis_dir is not None: 
+            vis(samples, targets, [points], vis_dir)
+        # accumulate MAE, MSE
+        mae = abs(predict_cnt - gt_cnt)
+        mse = (predict_cnt - gt_cnt) * (predict_cnt - gt_cnt)
+        maes.append(float(mae))
+        mses.append(float(mse))
+    # calc MAE, MSE
+    mae = np.mean(maes)
+    mse = np.sqrt(np.mean(mses))
+
+    return mae, mse

crowd_counter/models/__init__.py

@@ -0,0 +1,6 @@
+from .p2pnet import build
+
+# build the P2PNet model
+# set training to 'True' during training
+def build_model(args, training=False):
+    return build(args, training)

crowd_counter/models/backbone.py

@@ -0,0 +1,68 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+Backbone modules.
+"""
+from collections import OrderedDict
+
+import torch
+import torch.nn.functional as F
+import torchvision
+from torch import nn
+
+import crowd_counter.models.vgg_ as models
+
+class BackboneBase_VGG(nn.Module):
+    def __init__(self, backbone: nn.Module, num_channels: int, name: str, return_interm_layers: bool):
+        super().__init__()
+        features = list(backbone.features.children())
+        if return_interm_layers:
+            if name == 'vgg16_bn':
+                self.body1 = nn.Sequential(*features[:13])
+                self.body2 = nn.Sequential(*features[13:23])
+                self.body3 = nn.Sequential(*features[23:33])
+                self.body4 = nn.Sequential(*features[33:43])
+            else:
+                self.body1 = nn.Sequential(*features[:9])
+                self.body2 = nn.Sequential(*features[9:16])
+                self.body3 = nn.Sequential(*features[16:23])
+                self.body4 = nn.Sequential(*features[23:30])
+        else:
+            if name == 'vgg16_bn':
+                self.body = nn.Sequential(*features[:44])  # 16x down-sample
+            elif name == 'vgg16':
+                self.body = nn.Sequential(*features[:30])  # 16x down-sample
+        self.num_channels = num_channels
+        self.return_interm_layers = return_interm_layers
+
+    def forward(self, tensor_list):
+        out = []
+
+        if self.return_interm_layers:
+            xs = tensor_list
+            for _, layer in enumerate([self.body1, self.body2, self.body3, self.body4]):
+                xs = layer(xs)
+                out.append(xs)
+
+        else:
+            xs = self.body(tensor_list)
+            out.append(xs)
+        return out
+
+
+class Backbone_VGG(BackboneBase_VGG):
+    """ResNet backbone with frozen BatchNorm."""
+    def __init__(self, name: str, return_interm_layers: bool):
+        if name == 'vgg16_bn':
+            backbone = models.vgg16_bn(pretrained=True)
+        elif name == 'vgg16':
+            backbone = models.vgg16(pretrained=True)
+        num_channels = 256
+        super().__init__(backbone, num_channels, name, return_interm_layers)
+
+
+def build_backbone(args):
+    backbone = Backbone_VGG(args.backbone, True)
+    return backbone
+
+if __name__ == '__main__':
+    Backbone_VGG('vgg16', True)

crowd_counter/models/matcher.py

@@ -0,0 +1,83 @@
+
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+Mostly copy-paste from DETR (https://github.com/facebookresearch/detr).
+"""
+import torch
+from scipy.optimize import linear_sum_assignment
+from torch import nn
+
+
+class HungarianMatcher_Crowd(nn.Module):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    """
+
+    def __init__(self, cost_class: float = 1, cost_point: float = 1):
+        """Creates the matcher
+
+        Params:
+            cost_class: This is the relative weight of the foreground object
+            cost_point: This is the relative weight of the L1 error of the points coordinates in the matching cost
+        """
+        super().__init__()
+        self.cost_class = cost_class
+        self.cost_point = cost_point
+        assert cost_class != 0 or cost_point != 0, "all costs cant be 0"
+
+    @torch.no_grad()
+    def forward(self, outputs, targets):
+        """ Performs the matching
+
+        Params:
+            outputs: This is a dict that contains at least these entries:
+                 "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits
+                 "points": Tensor of dim [batch_size, num_queries, 2] with the predicted point coordinates
+
+            targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing:
+                 "labels": Tensor of dim [num_target_points] (where num_target_points is the number of ground-truth
+                           objects in the target) containing the class labels
+                 "points": Tensor of dim [num_target_points, 2] containing the target point coordinates
+
+        Returns:
+            A list of size batch_size, containing tuples of (index_i, index_j) where:
+                - index_i is the indices of the selected predictions (in order)
+                - index_j is the indices of the corresponding selected targets (in order)
+            For each batch element, it holds:
+                len(index_i) = len(index_j) = min(num_queries, num_target_points)
+        """
+        bs, num_queries = outputs["pred_logits"].shape[:2]
+
+        # We flatten to compute the cost matrices in a batch
+        out_prob = outputs["pred_logits"].flatten(0, 1).softmax(-1)  # [batch_size * num_queries, num_classes]
+        out_points = outputs["pred_points"].flatten(0, 1)  # [batch_size * num_queries, 2]
+
+        # Also concat the target labels and points
+        # tgt_ids = torch.cat([v["labels"] for v in targets])
+        tgt_ids = torch.cat([v["labels"] for v in targets])
+        tgt_points = torch.cat([v["point"] for v in targets])
+
+        # Compute the classification cost. Contrary to the loss, we don't use the NLL,
+        # but approximate it in 1 - proba[target class].
+        # The 1 is a constant that doesn't change the matching, it can be ommitted.
+        cost_class = -out_prob[:, tgt_ids]
+
+        # Compute the L2 cost between point
+        cost_point = torch.cdist(out_points, tgt_points, p=2)
+
+        # Compute the giou cost between point
+
+        # Final cost matrix
+        C = self.cost_point * cost_point + self.cost_class * cost_class
+        C = C.view(bs, num_queries, -1).cpu()
+
+        sizes = [len(v["point"]) for v in targets]
+        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))]
+        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]
+
+
+def build_matcher_crowd(args):
+    return HungarianMatcher_Crowd(cost_class=args.set_cost_class, cost_point=args.set_cost_point)

crowd_counter/models/p2pnet.py

@@ -0,0 +1,342 @@
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from crowd_counter.util.misc import (NestedTensor, nested_tensor_from_tensor_list,
+                       accuracy, get_world_size, interpolate,
+                       is_dist_avail_and_initialized)
+
+from .backbone import build_backbone
+from .matcher import build_matcher_crowd
+
+import numpy as np
+import time
+
+# the network frmawork of the regression branch
+class RegressionModel(nn.Module):
+    def __init__(self, num_features_in, num_anchor_points=4, feature_size=256):
+        super(RegressionModel, self).__init__()
+
+        self.conv1 = nn.Conv2d(num_features_in, feature_size, kernel_size=3, padding=1)
+        self.act1 = nn.ReLU()
+
+        self.conv2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act2 = nn.ReLU()
+
+        self.conv3 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act3 = nn.ReLU()
+
+        self.conv4 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act4 = nn.ReLU()
+
+        self.output = nn.Conv2d(feature_size, num_anchor_points * 2, kernel_size=3, padding=1)
+    # sub-branch forward
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.act1(out)
+
+        out = self.conv2(out)
+        out = self.act2(out)
+
+        out = self.output(out)
+
+        out = out.permute(0, 2, 3, 1)
+
+        return out.contiguous().view(out.shape[0], -1, 2)
+
+# the network frmawork of the classification branch
+class ClassificationModel(nn.Module):
+    def __init__(self, num_features_in, num_anchor_points=4, num_classes=80, prior=0.01, feature_size=256):
+        super(ClassificationModel, self).__init__()
+
+        self.num_classes = num_classes
+        self.num_anchor_points = num_anchor_points
+
+        self.conv1 = nn.Conv2d(num_features_in, feature_size, kernel_size=3, padding=1)
+        self.act1 = nn.ReLU()
+
+        self.conv2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act2 = nn.ReLU()
+
+        self.conv3 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act3 = nn.ReLU()
+
+        self.conv4 = nn.Conv2d(feature_size, feature_size, kernel_size=3, padding=1)
+        self.act4 = nn.ReLU()
+
+        self.output = nn.Conv2d(feature_size, num_anchor_points * num_classes, kernel_size=3, padding=1)
+        self.output_act = nn.Sigmoid()
+    # sub-branch forward
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.act1(out)
+
+        out = self.conv2(out)
+        out = self.act2(out)
+
+        out = self.output(out)
+
+        out1 = out.permute(0, 2, 3, 1)
+
+        batch_size, width, height, _ = out1.shape
+
+        out2 = out1.view(batch_size, width, height, self.num_anchor_points, self.num_classes)
+
+        return out2.contiguous().view(x.shape[0], -1, self.num_classes)
+
+# generate the reference points in grid layout
+def generate_anchor_points(stride=16, row=3, line=3):
+    row_step = stride / row
+    line_step = stride / line
+
+    shift_x = (np.arange(1, line + 1) - 0.5) * line_step - stride / 2
+    shift_y = (np.arange(1, row + 1) - 0.5) * row_step - stride / 2
+
+    shift_x, shift_y = np.meshgrid(shift_x, shift_y)
+
+    anchor_points = np.vstack((
+        shift_x.ravel(), shift_y.ravel()
+    )).transpose()
+
+    return anchor_points
+# shift the meta-anchor to get an acnhor points
+def shift(shape, stride, anchor_points):
+    shift_x = (np.arange(0, shape[1]) + 0.5) * stride
+    shift_y = (np.arange(0, shape[0]) + 0.5) * stride
+
+    shift_x, shift_y = np.meshgrid(shift_x, shift_y)
+
+    shifts = np.vstack((
+        shift_x.ravel(), shift_y.ravel()
+    )).transpose()
+
+    A = anchor_points.shape[0]
+    K = shifts.shape[0]
+    all_anchor_points = (anchor_points.reshape((1, A, 2)) + shifts.reshape((1, K, 2)).transpose((1, 0, 2)))
+    all_anchor_points = all_anchor_points.reshape((K * A, 2))
+
+    return all_anchor_points
+
+# this class generate all reference points on all pyramid levels
+class AnchorPoints(nn.Module):
+    def __init__(self, pyramid_levels=None, strides=None, row=3, line=3):
+        super(AnchorPoints, self).__init__()
+
+        if pyramid_levels is None:
+            self.pyramid_levels = [3, 4, 5, 6, 7]
+        else:
+            self.pyramid_levels = pyramid_levels
+
+        if strides is None:
+            self.strides = [2 ** x for x in self.pyramid_levels]
+
+        self.row = row
+        self.line = line
+
+    def forward(self, image):
+        image_shape = image.shape[2:]
+        image_shape = np.array(image_shape)
+        image_shapes = [(image_shape + 2 ** x - 1) // (2 ** x) for x in self.pyramid_levels]
+
+        all_anchor_points = np.zeros((0, 2)).astype(np.float32)
+        # get reference points for each level
+        for idx, p in enumerate(self.pyramid_levels):
+            anchor_points = generate_anchor_points(2**p, row=self.row, line=self.line)
+            shifted_anchor_points = shift(image_shapes[idx], self.strides[idx], anchor_points)
+            all_anchor_points = np.append(all_anchor_points, shifted_anchor_points, axis=0)
+
+        all_anchor_points = np.expand_dims(all_anchor_points, axis=0)
+        # send reference points to device
+        if torch.cuda.is_available():
+            return torch.from_numpy(all_anchor_points.astype(np.float32)).cuda()
+        else:
+            return torch.from_numpy(all_anchor_points.astype(np.float32))
+
+class Decoder(nn.Module):
+    def __init__(self, C3_size, C4_size, C5_size, feature_size=256):
+        super(Decoder, self).__init__()
+
+        # upsample C5 to get P5 from the FPN paper
+        self.P5_1 = nn.Conv2d(C5_size, feature_size, kernel_size=1, stride=1, padding=0)
+        self.P5_upsampled = nn.Upsample(scale_factor=2, mode='nearest')
+        self.P5_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=1, padding=1)
+
+        # add P5 elementwise to C4
+        self.P4_1 = nn.Conv2d(C4_size, feature_size, kernel_size=1, stride=1, padding=0)
+        self.P4_upsampled = nn.Upsample(scale_factor=2, mode='nearest')
+        self.P4_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=1, padding=1)
+
+        # add P4 elementwise to C3
+        self.P3_1 = nn.Conv2d(C3_size, feature_size, kernel_size=1, stride=1, padding=0)
+        self.P3_upsampled = nn.Upsample(scale_factor=2, mode='nearest')
+        self.P3_2 = nn.Conv2d(feature_size, feature_size, kernel_size=3, stride=1, padding=1)
+
+
+    def forward(self, inputs):
+        C3, C4, C5 = inputs
+
+        P5_x = self.P5_1(C5)
+        P5_upsampled_x = self.P5_upsampled(P5_x)
+        P5_x = self.P5_2(P5_x)
+
+        P4_x = self.P4_1(C4)
+        P4_x = P5_upsampled_x + P4_x
+        P4_upsampled_x = self.P4_upsampled(P4_x)
+        P4_x = self.P4_2(P4_x)
+
+        P3_x = self.P3_1(C3)
+        P3_x = P3_x + P4_upsampled_x
+        P3_x = self.P3_2(P3_x)
+
+        return [P3_x, P4_x, P5_x]
+
+# the defenition of the P2PNet model
+class P2PNet(nn.Module):
+    def __init__(self, backbone, row=2, line=2):
+        super().__init__()
+        self.backbone = backbone
+        self.num_classes = 2
+        # the number of all anchor points
+        num_anchor_points = row * line
+
+        self.regression = RegressionModel(num_features_in=256, num_anchor_points=num_anchor_points)
+        self.classification = ClassificationModel(num_features_in=256, \
+                                            num_classes=self.num_classes, \
+                                            num_anchor_points=num_anchor_points)
+
+        self.anchor_points = AnchorPoints(pyramid_levels=[3,], row=row, line=line)
+
+        self.fpn = Decoder(256, 512, 512)
+
+    def forward(self, samples: NestedTensor):
+        # get the backbone features
+        features = self.backbone(samples)
+        # forward the feature pyramid
+        features_fpn = self.fpn([features[1], features[2], features[3]])
+
+        batch_size = features[0].shape[0]
+        # run the regression and classification branch
+        regression = self.regression(features_fpn[1]) * 100 # 8x
+        classification = self.classification(features_fpn[1])
+        anchor_points = self.anchor_points(samples).repeat(batch_size, 1, 1)
+        # decode the points as prediction
+        output_coord = regression + anchor_points
+        output_class = classification
+        out = {'pred_logits': output_class, 'pred_points': output_coord}
+       
+        return out
+
+class SetCriterion_Crowd(nn.Module):
+
+    def __init__(self, num_classes, matcher, weight_dict, eos_coef, losses):
+        """ Create the criterion.
+        Parameters:
+            num_classes: number of object categories, omitting the special no-object category
+            matcher: module able to compute a matching between targets and proposals
+            weight_dict: dict containing as key the names of the losses and as values their relative weight.
+            eos_coef: relative classification weight applied to the no-object category
+            losses: list of all the losses to be applied. See get_loss for list of available losses.
+        """
+        super().__init__()
+        self.num_classes = num_classes
+        self.matcher = matcher
+        self.weight_dict = weight_dict
+        self.eos_coef = eos_coef
+        self.losses = losses
+        empty_weight = torch.ones(self.num_classes + 1)
+        empty_weight[0] = self.eos_coef
+        self.register_buffer('empty_weight', empty_weight)
+
+    def loss_labels(self, outputs, targets, indices, num_points):
+        """Classification loss (NLL)
+        targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes]
+        """
+        assert 'pred_logits' in outputs
+        src_logits = outputs['pred_logits']
+
+        idx = self._get_src_permutation_idx(indices)
+        target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)])
+        target_classes = torch.full(src_logits.shape[:2], 0,
+                                    dtype=torch.int64, device=src_logits.device)
+        target_classes[idx] = target_classes_o
+
+        loss_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.empty_weight)
+        losses = {'loss_ce': loss_ce}
+
+        return losses
+
+    def loss_points(self, outputs, targets, indices, num_points):
+
+        assert 'pred_points' in outputs
+        idx = self._get_src_permutation_idx(indices)
+        src_points = outputs['pred_points'][idx]
+        target_points = torch.cat([t['point'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+
+        loss_bbox = F.mse_loss(src_points, target_points, reduction='none')
+
+        losses = {}
+        losses['loss_point'] = loss_bbox.sum() / num_points
+
+        return losses
+
+    def _get_src_permutation_idx(self, indices):
+        # permute predictions following indices
+        batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)])
+        src_idx = torch.cat([src for (src, _) in indices])
+        return batch_idx, src_idx
+
+    def _get_tgt_permutation_idx(self, indices):
+        # permute targets following indices
+        batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)])
+        tgt_idx = torch.cat([tgt for (_, tgt) in indices])
+        return batch_idx, tgt_idx
+
+    def get_loss(self, loss, outputs, targets, indices, num_points, **kwargs):
+        loss_map = {
+            'labels': self.loss_labels,
+            'points': self.loss_points,
+        }
+        assert loss in loss_map, f'do you really want to compute {loss} loss?'
+        return loss_map[loss](outputs, targets, indices, num_points, **kwargs)
+
+    def forward(self, outputs, targets):
+        """ This performs the loss computation.
+        Parameters:
+             outputs: dict of tensors, see the output specification of the model for the format
+             targets: list of dicts, such that len(targets) == batch_size.
+                      The expected keys in each dict depends on the losses applied, see each loss' doc
+        """
+        output1 = {'pred_logits': outputs['pred_logits'], 'pred_points': outputs['pred_points']}
+
+        indices1 = self.matcher(output1, targets)
+
+        num_points = sum(len(t["labels"]) for t in targets)
+        num_points = torch.as_tensor([num_points], dtype=torch.float, device=next(iter(output1.values())).device)
+        if is_dist_avail_and_initialized():
+            torch.distributed.all_reduce(num_points)
+        num_boxes = torch.clamp(num_points / get_world_size(), min=1).item()
+
+        losses = {}
+        for loss in self.losses:
+            losses.update(self.get_loss(loss, output1, targets, indices1, num_boxes))
+
+        return losses
+
+# create the P2PNet model
+def build(args, training):
+    # treats persons as a single class
+    num_classes = 1
+
+    backbone = build_backbone(args)
+    model = P2PNet(backbone, args.row, args.line)
+    if not training: 
+        return model
+
+    weight_dict = {'loss_ce': 1, 'loss_points': args.point_loss_coef}
+    losses = ['labels', 'points']
+    matcher = build_matcher_crowd(args)
+    criterion = SetCriterion_Crowd(num_classes, \
+                                matcher=matcher, weight_dict=weight_dict, \
+                                eos_coef=args.eos_coef, losses=losses)
+
+    return model, criterion

crowd_counter/models/vgg_.py

@@ -0,0 +1,196 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+Mostly copy-paste from torchvision references.
+"""
+import torch
+import torch.nn as nn
+
+
+__all__ = [
+    'VGG', 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn',
+    'vgg19_bn', 'vgg19',
+]
+
+
+model_urls = {
+    'vgg11': 'https://download.pytorch.org/models/vgg11-bbd30ac9.pth',
+    'vgg13': 'https://download.pytorch.org/models/vgg13-c768596a.pth',
+    'vgg16': 'https://download.pytorch.org/models/vgg16-397923af.pth',
+    'vgg19': 'https://download.pytorch.org/models/vgg19-dcbb9e9d.pth',
+    'vgg11_bn': 'https://download.pytorch.org/models/vgg11_bn-6002323d.pth',
+    'vgg13_bn': 'https://download.pytorch.org/models/vgg13_bn-abd245e5.pth',
+    'vgg16_bn': 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth',
+    'vgg19_bn': 'https://download.pytorch.org/models/vgg19_bn-c79401a0.pth',
+}
+
+
+model_paths = {
+    'vgg16_bn': 'crowd_counter/weights/vgg16_bn-6c64b313.pth',
+    'vgg16': 'crowd_counter/weights/vgg16-397923af.pth',
+
+}
+
+
+class VGG(nn.Module):
+
+    def __init__(self, features, num_classes=1000, init_weights=True):
+        super(VGG, self).__init__()
+        self.features = features
+        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))
+        self.classifier = nn.Sequential(
+            nn.Linear(512 * 7 * 7, 4096),
+            nn.ReLU(True),
+            nn.Dropout(),
+            nn.Linear(4096, 4096),
+            nn.ReLU(True),
+            nn.Dropout(),
+            nn.Linear(4096, num_classes),
+        )
+        if init_weights:
+            self._initialize_weights()
+
+    def forward(self, x):
+        x = self.features(x)
+        x = self.avgpool(x)
+        x = torch.flatten(x, 1)
+        x = self.classifier(x)
+        return x
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+                if m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.constant_(m.bias, 0)
+
+
+def make_layers(cfg, batch_norm=False, sync=False):
+    layers = []
+    in_channels = 3
+    for v in cfg:
+        if v == 'M':
+            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
+        else:
+            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
+            if batch_norm:
+                if sync:
+                    print('use sync backbone')
+                    layers += [conv2d, nn.SyncBatchNorm(v), nn.ReLU(inplace=True)]
+                else:
+                    layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
+            else:
+                layers += [conv2d, nn.ReLU(inplace=True)]
+            in_channels = v
+    return nn.Sequential(*layers)
+
+
+cfgs = {
+    'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
+    'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],
+    'D': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],
+    'E': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],
+}
+
+
+def _vgg(arch, cfg, batch_norm, pretrained, progress, sync=False, **kwargs):
+    if pretrained:
+        kwargs['init_weights'] = False
+    model = VGG(make_layers(cfgs[cfg], batch_norm=batch_norm, sync=sync), **kwargs)
+    if pretrained:
+        state_dict = torch.load(model_paths[arch])
+        model.load_state_dict(state_dict)
+    return model
+
+
+def vgg11(pretrained=False, progress=True, **kwargs):
+    r"""VGG 11-layer model (configuration "A") from
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg11', 'A', False, pretrained, progress, **kwargs)
+
+
+def vgg11_bn(pretrained=False, progress=True, **kwargs):
+    r"""VGG 11-layer model (configuration "A") with batch normalization
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg11_bn', 'A', True, pretrained, progress, **kwargs)
+
+
+def vgg13(pretrained=False, progress=True, **kwargs):
+    r"""VGG 13-layer model (configuration "B")
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg13', 'B', False, pretrained, progress, **kwargs)
+
+
+def vgg13_bn(pretrained=False, progress=True, **kwargs):
+    r"""VGG 13-layer model (configuration "B") with batch normalization
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg13_bn', 'B', True, pretrained, progress, **kwargs)
+
+
+def vgg16(pretrained=False, progress=True, **kwargs):
+    r"""VGG 16-layer model (configuration "D")
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg16', 'D', False, pretrained, progress, **kwargs)
+
+
+def vgg16_bn(pretrained=False, progress=True, sync=False, **kwargs):
+    r"""VGG 16-layer model (configuration "D") with batch normalization
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg16_bn', 'D', True, pretrained, progress, sync=sync, **kwargs)
+
+
+def vgg19(pretrained=False, progress=True, **kwargs):
+    r"""VGG 19-layer model (configuration "E")
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg19', 'E', False, pretrained, progress, **kwargs)
+
+
+def vgg19_bn(pretrained=False, progress=True, **kwargs):
+    r"""VGG 19-layer model (configuration 'E') with batch normalization
+    `"Very Deep Convolutional Networks For Large-Scale Image Recognition" <https://arxiv.org/pdf/1409.1556.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _vgg('vgg19_bn', 'E', True, pretrained, progress, **kwargs)

crowd_counter/run_test.py

@@ -0,0 +1,104 @@
+import argparse
+import datetime
+import random
+import time
+from pathlib import Path
+
+import torch
+import torchvision.transforms as standard_transforms
+import numpy as np
+
+from PIL import Image
+import cv2
+from crowd_datasets import build_dataset
+from engine import *
+from models import build_model
+import os
+import warnings
+warnings.filterwarnings('ignore')
+
+def get_args_parser():
+    parser = argparse.ArgumentParser('Set parameters for P2PNet evaluation', add_help=False)
+    
+    # * Backbone
+    parser.add_argument('--backbone', default='vgg16_bn', type=str,
+                        help="name of the convolutional backbone to use")
+
+    parser.add_argument('--row', default=2, type=int,
+                        help="row number of anchor points")
+    parser.add_argument('--line', default=2, type=int,
+                        help="line number of anchor points")
+
+    parser.add_argument('--output_dir', default='',
+                        help='path where to save')
+    parser.add_argument('--weight_path', default='',
+                        help='path where the trained weights saved')
+
+    parser.add_argument('--gpu_id', default=0, type=int, help='the gpu used for evaluation')
+
+    return parser
+
+def main(args, img_path, debug=False):
+    os.environ["CUDA_VISIBLE_DEVICES"] = '{}'.format(args.gpu_id)
+    print(img_path)
+    device = torch.device('cuda')
+    # get the P2PNet
+    model = build_model(args)
+    # move to GPU
+    model.to(device)
+    # load trained model
+    if args.weight_path is not None:
+        checkpoint = torch.load(args.weight_path, map_location='cpu')
+        model.load_state_dict(checkpoint['model'])
+    # convert to eval mode
+    model.eval()
+    # create the pre-processing transform
+    transform = standard_transforms.Compose([
+        standard_transforms.ToTensor(), 
+        standard_transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+
+    # set your image path here
+    clean_img_path = img_path.split('/')[-1]
+
+    # load the images
+    img_raw = Image.open(img_path).convert('RGB')
+    # round the size
+    width, height = img_raw.size
+    new_width = width // 128 * 128
+    new_height = height // 128 * 128
+    img_raw = img_raw.resize((new_width, new_height), Image.ANTIALIAS)
+    # pre-proccessing
+    img = transform(img_raw)
+
+    samples = torch.Tensor(img).unsqueeze(0)
+    samples = samples.to(device)
+    # run inference
+    outputs = model(samples)
+    outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]
+
+    outputs_points = outputs['pred_points'][0]
+
+    threshold = 0.5
+    # filter the predictions
+    points = outputs_points[outputs_scores > threshold].detach().cpu().numpy().tolist()
+    predict_cnt = int((outputs_scores > threshold).sum())
+
+    outputs_scores = torch.nn.functional.softmax(outputs['pred_logits'], -1)[:, :, 1][0]
+
+    outputs_points = outputs['pred_points'][0]
+    # draw the predictions
+    size = 5
+    img_to_draw = cv2.cvtColor(np.array(img_raw), cv2.COLOR_RGB2BGR)
+    output_file = open(args.output_dir+"/"+clean_img_path, "w")
+    for p in points:
+        img_to_draw = cv2.circle(img_to_draw, (int(p[0]), int(p[1])), size, (0, 0, 255), -1)
+        output_file.write(str(p[0]) + " ")
+        output_file.write(str(p[1]) + "\n")
+    # save the visualized image
+    cv2.imwrite(os.path.join(args.output_dir, clean_img_path), img_to_draw)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('P2PNet evaluation script', parents=[get_args_parser()])
+    args = parser.parse_args()
+    main(args)

crowd_counter/train.py

@@ -0,0 +1,222 @@
+import argparse
+import datetime
+import random
+import time
+from pathlib import Path
+
+import torch
+from torch.utils.data import DataLoader, DistributedSampler
+
+from crowd_datasets import build_dataset
+from engine import *
+from models import build_model
+import os
+from tensorboardX import SummaryWriter
+import warnings
+warnings.filterwarnings('ignore')
+
+def get_args_parser():
+    parser = argparse.ArgumentParser('Set parameters for training P2PNet', add_help=False)
+    parser.add_argument('--lr', default=1e-4, type=float)
+    parser.add_argument('--lr_backbone', default=1e-5, type=float)
+    parser.add_argument('--batch_size', default=8, type=int)
+    parser.add_argument('--weight_decay', default=1e-4, type=float)
+    parser.add_argument('--epochs', default=3500, type=int)
+    parser.add_argument('--lr_drop', default=3500, type=int)
+    parser.add_argument('--clip_max_norm', default=0.1, type=float,
+                        help='gradient clipping max norm')
+
+    # Model parameters
+    parser.add_argument('--frozen_weights', type=str, default=None,
+                        help="Path to the pretrained model. If set, only the mask head will be trained")
+
+    # * Backbone
+    parser.add_argument('--backbone', default='vgg16_bn', type=str,
+                        help="Name of the convolutional backbone to use")
+
+    # * Matcher
+    parser.add_argument('--set_cost_class', default=1, type=float,
+                        help="Class coefficient in the matching cost")
+
+    parser.add_argument('--set_cost_point', default=0.05, type=float,
+                        help="L1 point coefficient in the matching cost")
+
+    # * Loss coefficients
+    parser.add_argument('--point_loss_coef', default=0.0002, type=float)
+
+    parser.add_argument('--eos_coef', default=0.5, type=float,
+                        help="Relative classification weight of the no-object class")
+    parser.add_argument('--row', default=2, type=int,
+                        help="row number of anchor points")
+    parser.add_argument('--line', default=2, type=int,
+                        help="line number of anchor points")
+
+    # dataset parameters
+    parser.add_argument('--dataset_file', default='SHHA')
+    parser.add_argument('--data_root', default='./new_public_density_data',
+                        help='path where the dataset is')
+    
+    parser.add_argument('--output_dir', default='./log',
+                        help='path where to save, empty for no saving')
+    parser.add_argument('--checkpoints_dir', default='./ckpt',
+                        help='path where to save checkpoints, empty for no saving')
+    parser.add_argument('--tensorboard_dir', default='./runs',
+                        help='path where to save, empty for no saving')
+
+    parser.add_argument('--seed', default=42, type=int)
+    parser.add_argument('--resume', default='', help='resume from checkpoint')
+    parser.add_argument('--start_epoch', default=0, type=int, metavar='N',
+                        help='start epoch')
+    parser.add_argument('--eval', action='store_true')
+    parser.add_argument('--num_workers', default=8, type=int)
+    parser.add_argument('--eval_freq', default=5, type=int,
+                        help='frequency of evaluation, default setting is evaluating in every 5 epoch')
+    parser.add_argument('--gpu_id', default=0, type=int, help='the gpu used for training')
+
+    return parser
+
+def main(args):
+    os.environ["CUDA_VISIBLE_DEVICES"] = '{}'.format(args.gpu_id)
+    # create the logging file
+    run_log_name = os.path.join(args.output_dir, 'run_log.txt')
+    with open(run_log_name, "w") as log_file:
+        log_file.write('Eval Log %s\n' % time.strftime("%c"))
+
+    if args.frozen_weights is not None:
+        assert args.masks, "Frozen training is meant for segmentation only"
+    # backup the arguments
+    print(args)
+    with open(run_log_name, "a") as log_file:
+        log_file.write("{}".format(args))
+    device = torch.device('cuda')
+    # fix the seed for reproducibility
+    seed = args.seed + utils.get_rank()
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    # get the P2PNet model
+    model, criterion = build_model(args, training=True)
+    # move to GPU
+    model.to(device)
+    criterion.to(device)
+
+    model_without_ddp = model
+
+    n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print('number of params:', n_parameters)
+    # use different optimation params for different parts of the model
+    param_dicts = [
+        {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad]},
+        {
+            "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad],
+            "lr": args.lr_backbone,
+        },
+    ]
+    # Adam is used by default
+    optimizer = torch.optim.Adam(param_dicts, lr=args.lr)
+    lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop)
+    # create the dataset
+    loading_data = build_dataset(args=args)
+    # create the training and valiation set
+    train_set, val_set = loading_data(args.data_root)
+    # create the sampler used during training
+    sampler_train = torch.utils.data.RandomSampler(train_set)
+    sampler_val = torch.utils.data.SequentialSampler(val_set)
+
+    batch_sampler_train = torch.utils.data.BatchSampler(
+        sampler_train, args.batch_size, drop_last=True)
+    # the dataloader for training
+    data_loader_train = DataLoader(train_set, batch_sampler=batch_sampler_train,
+                                   collate_fn=utils.collate_fn_crowd, num_workers=args.num_workers)
+
+    data_loader_val = DataLoader(val_set, 1, sampler=sampler_val,
+                                    drop_last=False, collate_fn=utils.collate_fn_crowd, num_workers=args.num_workers)
+
+    if args.frozen_weights is not None:
+        checkpoint = torch.load(args.frozen_weights, map_location='cpu')
+        model_without_ddp.detr.load_state_dict(checkpoint['model'])
+    # resume the weights and training state if exists
+    if args.resume:
+        checkpoint = torch.load(args.resume, map_location='cpu')
+        model_without_ddp.load_state_dict(checkpoint['model'])
+        if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint:
+            optimizer.load_state_dict(checkpoint['optimizer'])
+            lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
+            args.start_epoch = checkpoint['epoch'] + 1
+
+    print("Start training")
+    start_time = time.time()
+    # save the performance during the training
+    mae = []
+    mse = []
+    # the logger writer
+    writer = SummaryWriter(args.tensorboard_dir)
+    
+    step = 0
+    # training starts here
+    for epoch in range(args.start_epoch, args.epochs):
+        t1 = time.time()
+        stat = train_one_epoch(
+            model, criterion, data_loader_train, optimizer, device, epoch,
+            args.clip_max_norm)
+
+        # record the training states after every epoch
+        if writer is not None:
+            with open(run_log_name, "a") as log_file:
+                log_file.write("loss/loss@{}: {}".format(epoch, stat['loss']))
+                log_file.write("loss/loss_ce@{}: {}".format(epoch, stat['loss_ce']))
+                
+            writer.add_scalar('loss/loss', stat['loss'], epoch)
+            writer.add_scalar('loss/loss_ce', stat['loss_ce'], epoch)
+
+        t2 = time.time()
+        print('[ep %d][lr %.7f][%.2fs]' % \
+              (epoch, optimizer.param_groups[0]['lr'], t2 - t1))
+        with open(run_log_name, "a") as log_file:
+            log_file.write('[ep %d][lr %.7f][%.2fs]' % (epoch, optimizer.param_groups[0]['lr'], t2 - t1))
+        # change lr according to the scheduler
+        lr_scheduler.step()
+        # save latest weights every epoch
+        checkpoint_latest_path = os.path.join(args.checkpoints_dir, 'latest.pth')
+        torch.save({
+            'model': model_without_ddp.state_dict(),
+        }, checkpoint_latest_path)
+        # run evaluation
+        if epoch % args.eval_freq == 0 and epoch != 0:
+            t1 = time.time()
+            result = evaluate_crowd_no_overlap(model, data_loader_val, device)
+            t2 = time.time()
+
+            mae.append(result[0])
+            mse.append(result[1])
+            # print the evaluation results
+            print('=======================================test=======================================')
+            print("mae:", result[0], "mse:", result[1], "time:", t2 - t1, "best mae:", np.min(mae), )
+            with open(run_log_name, "a") as log_file:
+                log_file.write("mae:{}, mse:{}, time:{}, best mae:{}".format(result[0], 
+                                result[1], t2 - t1, np.min(mae)))
+            print('=======================================test=======================================')
+            # recored the evaluation results
+            if writer is not None:
+                with open(run_log_name, "a") as log_file:
+                    log_file.write("metric/mae@{}: {}".format(step, result[0]))
+                    log_file.write("metric/mse@{}: {}".format(step, result[1]))
+                writer.add_scalar('metric/mae', result[0], step)
+                writer.add_scalar('metric/mse', result[1], step)
+                step += 1
+
+            # save the best model since begining
+            if abs(np.min(mae) - result[0]) < 0.01:
+                checkpoint_best_path = os.path.join(args.checkpoints_dir, 'best_mae.pth')
+                torch.save({
+                    'model': model_without_ddp.state_dict(),
+                }, checkpoint_best_path)
+    # total time for training
+    total_time = time.time() - start_time
+    total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+    print('Training time {}'.format(total_time_str))
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('P2PNet training and evaluation script', parents=[get_args_parser()])
+    args = parser.parse_args()
+    main(args)

crowd_counter/util/misc.py

@@ -0,0 +1,517 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+Misc functions, including distributed helpers.
+
+Mostly copy-paste from torchvision references.
+"""
+import os
+import subprocess
+import time
+from collections import defaultdict, deque
+import datetime
+import pickle
+from typing import Optional, List
+
+import torch
+import torch.distributed as dist
+from torch import Tensor
+
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+# needed due to empty tensor bug in pytorch and torchvision 0.5
+import torchvision
+if float(torchvision.__version__[:3]) < 0.7:
+    pass
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.4f} ({global_avg:.4f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda')
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value)
+
+
+def all_gather(data):
+    """
+    Run all_gather on arbitrary picklable data (not necessarily tensors)
+    Args:
+        data: any picklable object
+    Returns:
+        list[data]: list of data gathered from each rank
+    """
+    world_size = get_world_size()
+    if world_size == 1:
+        return [data]
+
+    # serialized to a Tensor
+    buffer = pickle.dumps(data)
+    storage = torch.ByteStorage.from_buffer(buffer)
+    tensor = torch.ByteTensor(storage).to("cuda")
+
+    # obtain Tensor size of each rank
+    local_size = torch.tensor([tensor.numel()], device="cuda")
+    size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)]
+    dist.all_gather(size_list, local_size)
+    size_list = [int(size.item()) for size in size_list]
+    max_size = max(size_list)
+
+    # receiving Tensor from all ranks
+    # we pad the tensor because torch all_gather does not support
+    # gathering tensors of different shapes
+    tensor_list = []
+    for _ in size_list:
+        tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda"))
+    if local_size != max_size:
+        padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda")
+        tensor = torch.cat((tensor, padding), dim=0)
+    dist.all_gather(tensor_list, tensor)
+
+    data_list = []
+    for size, tensor in zip(size_list, tensor_list):
+        buffer = tensor.cpu().numpy().tobytes()[:size]
+        data_list.append(pickle.loads(buffer))
+
+    return data_list
+
+
+def reduce_dict(input_dict, average=True):
+    """
+    Args:
+        input_dict (dict): all the values will be reduced
+        average (bool): whether to do average or sum
+    Reduce the values in the dictionary from all processes so that all processes
+    have the averaged results. Returns a dict with the same fields as
+    input_dict, after reduction.
+    """
+    world_size = get_world_size()
+    if world_size < 2:
+        return input_dict
+    with torch.no_grad():
+        names = []
+        values = []
+        # sort the keys so that they are consistent across processes
+        for k in sorted(input_dict.keys()):
+            names.append(k)
+            values.append(input_dict[k])
+        values = torch.stack(values, dim=0)
+        dist.all_reduce(values)
+        if average:
+            values /= world_size
+        reduced_dict = {k: v for k, v in zip(names, values)}
+    return reduced_dict
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(
+            type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append(
+                "{}: {}".format(name, str(meter))
+            )
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ''
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt='{avg:.4f}')
+        data_time = SmoothedValue(fmt='{avg:.4f}')
+        space_fmt = ':' + str(len(str(len(iterable)))) + 'd'
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}',
+                'max mem: {memory:.0f}'
+            ])
+        else:
+            log_msg = self.delimiter.join([
+                header,
+                '[{0' + space_fmt + '}/{1}]',
+                'eta: {eta}',
+                '{meters}',
+                'time: {time}',
+                'data: {data}'
+            ])
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time),
+                        memory=torch.cuda.max_memory_allocated() / MB))
+                else:
+                    print(log_msg.format(
+                        i, len(iterable), eta=eta_string,
+                        meters=str(self),
+                        time=str(iter_time), data=str(data_time)))
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print('{} Total time: {} ({:.4f} s / it)'.format(
+            header, total_time_str, total_time / len(iterable)))
+
+
+def get_sha():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+
+    def _run(command):
+        return subprocess.check_output(command, cwd=cwd).decode('ascii').strip()
+    sha = 'N/A'
+    diff = "clean"
+    branch = 'N/A'
+    try:
+        sha = _run(['git', 'rev-parse', 'HEAD'])
+        subprocess.check_output(['git', 'diff'], cwd=cwd)
+        diff = _run(['git', 'diff-index', 'HEAD'])
+        diff = "has uncommited changes" if diff else "clean"
+        branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
+    except Exception:
+        pass
+    message = f"sha: {sha}, status: {diff}, branch: {branch}"
+    return message
+
+
+def collate_fn(batch):
+    batch = list(zip(*batch))
+    batch[0] = nested_tensor_from_tensor_list(batch[0])
+    return tuple(batch)
+
+def collate_fn_crowd(batch):
+    # re-organize the batch
+    batch_new = []
+    for b in batch:
+        imgs, points = b
+        if imgs.ndim == 3:
+            imgs = imgs.unsqueeze(0)
+        for i in range(len(imgs)):
+            batch_new.append((imgs[i, :, :, :], points[i]))
+    batch = batch_new
+    batch = list(zip(*batch))
+    batch[0] = nested_tensor_from_tensor_list(batch[0])
+    return tuple(batch)
+
+
+def _max_by_axis(the_list):
+    # type: (List[List[int]]) -> List[int]
+    maxes = the_list[0]
+    for sublist in the_list[1:]:
+        for index, item in enumerate(sublist):
+            maxes[index] = max(maxes[index], item)
+    return maxes
+
+def _max_by_axis_pad(the_list):
+    # type: (List[List[int]]) -> List[int]
+    maxes = the_list[0]
+    for sublist in the_list[1:]:
+        for index, item in enumerate(sublist):
+            maxes[index] = max(maxes[index], item)
+
+    block = 128
+
+    for i in range(2):
+        maxes[i+1] = ((maxes[i+1] - 1) // block + 1) * block
+    return maxes
+
+
+def nested_tensor_from_tensor_list(tensor_list: List[Tensor]):
+    # TODO make this more general
+    if tensor_list[0].ndim == 3:
+
+        # TODO make it support different-sized images
+        max_size = _max_by_axis_pad([list(img.shape) for img in tensor_list])
+        # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list]))
+        batch_shape = [len(tensor_list)] + max_size
+        b, c, h, w = batch_shape
+        dtype = tensor_list[0].dtype
+        device = tensor_list[0].device
+        tensor = torch.zeros(batch_shape, dtype=dtype, device=device)
+        for img, pad_img in zip(tensor_list, tensor):
+            pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img)
+    else:
+        raise ValueError('not supported')
+    return tensor
+
+class NestedTensor(object):
+    def __init__(self, tensors, mask: Optional[Tensor]):
+        self.tensors = tensors
+        self.mask = mask
+
+    def to(self, device):
+        # type: (Device) -> NestedTensor # noqa
+        cast_tensor = self.tensors.to(device)
+        mask = self.mask
+        if mask is not None:
+            assert mask is not None
+            cast_mask = mask.to(device)
+        else:
+            cast_mask = None
+        return NestedTensor(cast_tensor, cast_mask)
+
+    def decompose(self):
+        return self.tensors, self.mask
+
+    def __repr__(self):
+        return str(self.tensors)
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop('force', False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def get_world_size():
+    if not is_dist_avail_and_initialized():
+        return 1
+    return dist.get_world_size()
+
+
+def get_rank():
+    if not is_dist_avail_and_initialized():
+        return 0
+    return dist.get_rank()
+
+
+def is_main_process():
+    return get_rank() == 0
+
+
+def save_on_master(*args, **kwargs):
+    if is_main_process():
+        torch.save(*args, **kwargs)
+
+
+def init_distributed_mode(args):
+    if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+        args.rank = int(os.environ["RANK"])
+        args.world_size = int(os.environ['WORLD_SIZE'])
+        args.gpu = int(os.environ['LOCAL_RANK'])
+    elif 'SLURM_PROCID' in os.environ:
+        args.rank = int(os.environ['SLURM_PROCID'])
+        args.gpu = args.rank % torch.cuda.device_count()
+    else:
+        print('Not using distributed mode')
+        args.distributed = False
+        return
+
+    args.distributed = True
+
+    torch.cuda.set_device(args.gpu)
+    args.dist_backend = 'nccl'
+    print('| distributed init (rank {}): {}'.format(
+        args.rank, args.dist_url), flush=True)
+    torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
+                                         world_size=args.world_size, rank=args.rank)
+    torch.distributed.barrier()
+    setup_for_distributed(args.rank == 0)
+
+
+@torch.no_grad()
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    if target.numel() == 0:
+        return [torch.zeros([], device=output.device)]
+    maxk = max(topk)
+    batch_size = target.size(0)
+
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].view(-1).float().sum(0)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res
+
+
+def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None):
+    # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor
+    """
+    Equivalent to nn.functional.interpolate, but with support for empty batch sizes.
+    This will eventually be supported natively by PyTorch, and this
+    class can go away.
+    """
+    if float(torchvision.__version__[:3]) < 0.7:
+        if input.numel() > 0:
+            return torch.nn.functional.interpolate(
+                input, size, scale_factor, mode, align_corners
+            )
+
+        output_shape = _output_size(2, input, size, scale_factor)
+        output_shape = list(input.shape[:-2]) + list(output_shape)
+        return _new_empty_tensor(input, output_shape)
+    else:
+        return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners)
+
+
+class FocalLoss(nn.Module):
+    r"""
+        This criterion is a implemenation of Focal Loss, which is proposed in
+        Focal Loss for Dense Object Detection.
+
+            Loss(x, class) = - \alpha (1-softmax(x)[class])^gamma \log(softmax(x)[class])
+
+        The losses are averaged across observations for each minibatch.
+
+        Args:
+            alpha(1D Tensor, Variable) : the scalar factor for this criterion
+            gamma(float, double) : gamma > 0; reduces the relative loss for well-classified examples (p > .5),
+                                   putting more focus on hard, misclassified examples
+            size_average(bool): By default, the losses are averaged over observations for each minibatch.
+                                However, if the field size_average is set to False, the losses are
+                                instead summed for each minibatch.
+
+
+    """
+    def __init__(self, class_num, alpha=None, gamma=2, size_average=True):
+        super(FocalLoss, self).__init__()
+        if alpha is None:
+            self.alpha = Variable(torch.ones(class_num, 1))
+        else:
+            if isinstance(alpha, Variable):
+                self.alpha = alpha
+            else:
+                self.alpha = Variable(alpha)
+        self.gamma = gamma
+        self.class_num = class_num
+        self.size_average = size_average
+
+    def forward(self, inputs, targets):
+        N = inputs.size(0)
+        C = inputs.size(1)
+        P = F.softmax(inputs)
+
+        class_mask = inputs.data.new(N, C).fill_(0)
+        class_mask = Variable(class_mask)
+        ids = targets.view(-1, 1)
+        class_mask.scatter_(1, ids.data, 1.)
+
+        if inputs.is_cuda and not self.alpha.is_cuda:
+            self.alpha = self.alpha.cuda()
+        alpha = self.alpha[ids.data.view(-1)]
+
+        probs = (P*class_mask).sum(1).view(-1,1)
+
+        log_p = probs.log()
+        batch_loss = -alpha*(torch.pow((1-probs), self.gamma))*log_p
+
+        if self.size_average:
+            loss = batch_loss.mean()
+        else:
+            loss = batch_loss.sum()
+        return loss

crowd_counter/weights/SHTechA.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:506047732b128ff09efef18e94bfacbe35fcfef300e5e9eeeece259b0488c63f
+size 86372926

requirements.txt

@@ -0,0 +1,11 @@
+torch
+torchvision
+tensorboardX
+easydict
+pandas
+numpy
+scipy
+matplotlib
+Pillow
+opencv-python
+gradio==3.50