diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..b4a176e3a249a06e10e4a905492b4ba2ba94479e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/ConvTransformer-Pipeline.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/failure_case.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/marconet_vs_marconetplus.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/pipeline.png filter=lfs diff=lfs merge=lfs -text +Imgs/prior.gif filter=lfs diff=lfs merge=lfs -text +Imgs/text_line_sr.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/w-interpolation.gif filter=lfs diff=lfs merge=lfs -text +Imgs/whole_1.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/whole_2.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/whole_3.jpg filter=lfs diff=lfs merge=lfs -text +Imgs/whole_4.jpg filter=lfs diff=lfs merge=lfs -text +Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_12.JPG filter=lfs diff=lfs merge=lfs -text +Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_16.JPG filter=lfs diff=lfs merge=lfs -text +Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_8.JPG filter=lfs diff=lfs merge=lfs -text +Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_30.JPG filter=lfs diff=lfs merge=lfs -text +Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_62.JPG filter=lfs diff=lfs merge=lfs -text diff --git a/Imgs/failure_case.jpg b/Imgs/failure_case.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f104ead8ff97479f3e8f472e3583663a02404d17 --- /dev/null +++ b/Imgs/failure_case.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de31fc543a29ccac3a412eb68d972e1934968ef1837748bff3085e98142bdc13 +size 376530 diff --git a/Imgs/marconet_vs_marconetplus.jpg b/Imgs/marconet_vs_marconetplus.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9b02960edc70e98797fb99392e5dab62138abeb3 --- /dev/null +++ b/Imgs/marconet_vs_marconetplus.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9404f740883da35363403ead59424973e08c45dd0e2c5d4ea936de0f91aeb156 +size 1780995 diff --git a/Imgs/pipeline.png b/Imgs/pipeline.png new file mode 100644 index 0000000000000000000000000000000000000000..80d2457b89b7e7de566790ecae162228746d3a1f --- /dev/null +++ b/Imgs/pipeline.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f21f02cf5a1ab32880f17715a6d0ff374a9574e3f9bf3ac319278494d5a1b26d +size 1791820 diff --git a/Imgs/prior.gif b/Imgs/prior.gif new file mode 100644 index 0000000000000000000000000000000000000000..2844287838c3f90386c10a18d3ff5031c3fc6f94 --- /dev/null +++ b/Imgs/prior.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:727c6520d0eddabcc3d3b6eddf13a4c48594c4d6b467ae34c0dbbcf7eaa3de74 +size 4208074 diff --git a/Imgs/text_line_sr.jpg b/Imgs/text_line_sr.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1a24d5dceae30ff3e0d18bc0dc422ef56fc37027 --- /dev/null +++ b/Imgs/text_line_sr.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:934d345323d54e00ff9b5b5a301f22d3d0a93ea83357aeafc6176d3e312096f7 +size 100731 diff --git a/Imgs/w-interpolation.gif b/Imgs/w-interpolation.gif new file mode 100644 index 0000000000000000000000000000000000000000..6b14c434d28187781c3e147b5477f9856bb6f082 --- /dev/null +++ b/Imgs/w-interpolation.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e0e259b9326d9626913ce87f00449e2c66402e8548d1e5f95ad718e653aafb +size 500301 diff --git a/Imgs/whole_1.jpg b/Imgs/whole_1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e14cbc42169fdd0dd16edca0ba27143f99475553 --- /dev/null +++ b/Imgs/whole_1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9024a701f41a8e042a93fdc90c0d2aa530fb38dd21c6b6dbd6a957d7adef1fc +size 204543 diff --git a/Imgs/whole_2.jpg b/Imgs/whole_2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ca8f9285763ca3dc5f1ea336f52f9386ba37d1da --- /dev/null +++ b/Imgs/whole_2.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99acdfdcdf8edeb7625899ea74e2638fa1d80fd7d15a4967f0388977e6ace502 +size 164645 diff --git a/Imgs/whole_3.jpg b/Imgs/whole_3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0715dfbb12af8af016f7ff9e0c4472a1ad6d93f8 --- /dev/null +++ b/Imgs/whole_3.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3503da8616151e0fbcdf16c882c450722e5d51450db902d9b687728dfdb651 +size 173745 diff --git a/Imgs/whole_4.jpg b/Imgs/whole_4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f96a59c1bf85fde925c20cfea526a403bcd0ca2 --- /dev/null +++ b/Imgs/whole_4.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97e8ff652b0845d5b474358bb2f0c602c891aad7f523f38512b814123ec44693 +size 139409 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..80dabb7099fe0c3d9d4678d88c7256c7184f5fef --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +S-Lab License 1.0 + +Copyright 2025 S-Lab +Redistribution and use for non-commercial purpose in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +4. In the event that redistribution and/or use for commercial purpose in source or binary forms, with or without modification is required, please contact the contributor(s) of the work. diff --git a/README.md b/README.md index 5f60e886b11b778c440fd43478e40408db3d8951..34df4dac6cb1fb7937354fa4d2eed4a89c2f6dad 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,147 @@ ---- -title: Marconetplusplus -emoji: 🦀 -colorFrom: yellow -colorTo: yellow -sdk: gradio -sdk_version: 5.42.0 -app_file: app.py -pinned: false -license: cc-by-nc-4.0 -short_description: Chinese Text Image Super-Resolution ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference + +
+ + ## [Enhanced Generative Structure Prior for Chinese Text Image Super-Resolution](https://arxiv.org/pdf/2508.07537) + +[Xiaoming Li](https://csxmli2016.github.io/), [Wangmeng Zuo](https://scholar.google.com/citations?hl=en&user=rUOpCEYAAAAJ&view_op=list_works), [Chen Change Loy](https://www.mmlab-ntu.com/person/ccloy/) + +S-Lab, Nanyang Technological University + +
+ + +### 👷 The whole framework: +
+ +
+ + + + + +### 👷 Character Structure Prior Pretraining: +
+ +
+ + +## 🔔 MARCONet 🆚 MARCONet++ +> - MARCONet is designed for **regular character layout** only. See details of [MARCONet](https://github.com/csxmli2016/MARCONet). +> - MARCONet++ has more accurate alignment between character structural prior (green structure) and the degraded image. +
+ +
+ + + +## 📋 TODO +- [x] Release the inference code and model. +- [ ] Release the training code (no plans to release for now). + + +## 🚶 Getting Started + +``` +git clone https://github.com/csxmli2016/MARCONetPlusPlus +cd MARCONetPlusPlus +conda create -n mplus python=3.8 -y +conda activate mplus +pip install -r requirements.txt +``` + +## 🚶 Inference +Download the pre-trained models +``` +python utils/download_github.py +``` + +and run for restoring **text lines:** +``` +CUDA_VISIBLE_DEVICES=0 python test_marconetplus.py -i ./Testsets/LR_TextLines -a -s +``` +or run for restoring **the whole text image:** +``` +CUDA_VISIBLE_DEVICES=0 python test_marconetplus.py -i ./Testsets/LR_Whole -b -s -f 2 +``` + +``` +# Parameters: +-i: --input_path, default: ./Testsets/LR_TextLines or ./Testsets/LR_TextWhole +-o: --output_path, default: None will automatically make the saving dir with the format of '[LR path]_TIME_MARCONetPlus' +-a: --aligned, if the input is text lines, use -a; otherwise, the input is the whole text image and needs text line detection, do not use -a +-b: --bg_sr, when restoring the whole text images, use -b to restore the background region with BSRGAN. Without -b, background will keep the same as input +-f: --factor_scale, default: 2. When restoring the whole text images, use -f to define the scale factor of output +-s: --save_text, if you want to see the details of prior alignment, predicted characters, and locations, use -s +``` + + + +## 🏃 Restoring Real-world Chinese Text Images +> - We use [BSRGAN](https://github.com/cszn/BSRGAN) to restore the background region. +> - The parameters are tested on an NVIDIA A100 GPU (40G). +> - ⚠️ If the inference speed is slow, this is caused by the large size of the input text image or the large factor_scale. You can resize it based on your needs. + +[](https://imgsli.com/NDA2MDUw) [](https://imgsli.com/NDA2MDYw) + +[](https://imgsli.com/NDA2MTE0) [](https://imgsli.com/NDA2MDYy) + + +## 🏃 Restoring detected text line + + + + +

🏃 Style w interpolation from three characters with different styles

 + +
+ + + + +## ‼️ Failure Case +Despite its high-fidelity performance, MARCONet++ still struggles in some real-world scenarios as it highly relies on: + +- Real world character **Recognition** on complex degraded text images +- Real world character **Detection** on complex degraded text images +- Text line detection and segmentation +- Domain gap between our synthetic and real-world text images + + + + +> 🍒 Restoring complex character with high fidelity under such conditions has significant challenges. +We have also explored various approaches, such as training OCR models with Transformers and using YOLO or Transformer-based methods for character detection, but these methods generally encounter the same issues. +We encourage any potential collaborations to jointly tackle this challenge and advance robust, high-fidelity text restoration. + + +## 📎 RealCE-1K benchmark +To quantitatively evaluate on real-world Chinese text line images, we curate a benchmark by filtering the [RealCE](https://github.com/mjq11302010044/Real-CE) test set to exclude images containing multiple text lines or inaccurate annotations, thereby constructing a Chinese text SR benchmark (see Section IV.B of our paper). You can download the RealCE-1K benchmark from [here](https://github.com/csxmli2016/MARCONetPlusPlus/releases/download/v1/RealCE-1K.zip). + +## 🍺 Acknowledgement +This project is built based on the excellent [KAIR](https://github.com/cszn/KAIR) and [RealCE](https://github.com/mjq11302010044/Real-CE). + +## ©️ License +This project is licensed under NTU S-Lab License 1.0. Redistribution and use should follow this license. + + +## 🍻 Citation +``` +@article{li2025marconetplus, + author = {Li, Xiaoming and Zuo, Wangmeng and Loy, Chen Change}, + title = {Enhanced Generative Structure Prior for Chinese Text Image Super-Resolution}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, + year = {2025} +} + +@InProceedings{li2023marconet, + author = {Li, Xiaoming and Zuo, Wangmeng and Loy, Chen Change}, + title = {Learning Generative Structure Prior for Blind Text Image Super-resolution}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year = {2023} +} +``` + + diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_15_34.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_15_34.JPG new file mode 100644 index 0000000000000000000000000000000000000000..16e67e36451d9283b1343dc2ed86e31adf0e07c8 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_15_34.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_32_1.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_32_1.JPG new file mode 100644 index 0000000000000000000000000000000000000000..ac4e2bc2bd8d3743d6fa299a928c247fc33e284e Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_32_1.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_3_7.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_3_7.JPG new file mode 100644 index 0000000000000000000000000000000000000000..c4ebf18f6b2db8c2834a0840dff96553d041a4c3 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_3_7.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_87_7.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_87_7.JPG new file mode 100644 index 0000000000000000000000000000000000000000..9964b5e602b4c6af804822ab771cf4ef0e0a94ce Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_87_7.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_15.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_15.JPG new file mode 100644 index 0000000000000000000000000000000000000000..5fc2f2cb99e152389e12ee366226e6b337864072 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_15.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_21.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_21.JPG new file mode 100644 index 0000000000000000000000000000000000000000..54808ac9e7cdabee96aa5bf97f6ebe03cafb8d6a Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_21.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_8.JPG b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_8.JPG new file mode 100644 index 0000000000000000000000000000000000000000..10588875891e4cf44e6ef3a53ec904ad38507bc2 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_TextSR_renew_88_8.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_10.JPG b/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_10.JPG new file mode 100644 index 0000000000000000000000000000000000000000..67f2f0933997e5ba9cd4c6f2b06acb43cf7c31cc Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_10.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_12.JPG b/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_12.JPG new file mode 100644 index 0000000000000000000000000000000000000000..b1d0ad054f4a538642c56f883588b3ebe6ca236b Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_testSR_2101225_xwm_renew_8_12.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_28_19.JPG b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_28_19.JPG new file mode 100644 index 0000000000000000000000000000000000000000..c6779a37e8e25c55353851c04260c0f775d6c0b0 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_28_19.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_62_5.JPG b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_62_5.JPG new file mode 100644 index 0000000000000000000000000000000000000000..7e3605aa4fff304b32c7225ee4dd0591b982cf03 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_62_5.JPG differ diff --git a/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_78_8.JPG b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_78_8.JPG new file mode 100644 index 0000000000000000000000000000000000000000..6fe8eba24453fb43cbb3554e5661d1375377a982 Binary files /dev/null and b/Testsets/LR_TextLines/ip11pro_output_textSR_211206_renew_78_8.JPG differ diff --git a/Testsets/LR_TextLines/test_long_tmp.jpg b/Testsets/LR_TextLines/test_long_tmp.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7efc2ed8d400014d983bd2c19d9dd7e58ad5d879 Binary files /dev/null and b/Testsets/LR_TextLines/test_long_tmp.jpg differ diff --git a/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_12.JPG b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_12.JPG new file mode 100644 index 0000000000000000000000000000000000000000..c63c8d2101d931e3be250de5d7d74d3fe0b31216 --- /dev/null +++ b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_12.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3c34bd8fcf9b0a77b122b120f63b44ea4cf05c8fae88378103c03d55ff52d3 +size 558137 diff --git a/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_16.JPG b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_16.JPG new file mode 100644 index 0000000000000000000000000000000000000000..109f0946a4d87fe8847aacd79651063c1272b546 --- /dev/null +++ b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_16.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d692cd59c8fd006265fa96457de731ce499e6e818094e346da5639158003c4d +size 545834 diff --git a/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_8.JPG b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_8.JPG new file mode 100644 index 0000000000000000000000000000000000000000..973579f79f6478318bd1ea1951459edb00fee3dd --- /dev/null +++ b/Testsets/LR_Whole/ip11pro_output_testSR_2101225_xwm_renew_8.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af607dbd64819c0b94c74393f827cbfb2cea39832e1805b073e2ce0821e6d58 +size 1514404 diff --git a/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_30.JPG b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_30.JPG new file mode 100644 index 0000000000000000000000000000000000000000..8e0049e1c57fb0eb7123c950075554566d037390 --- /dev/null +++ b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_30.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6df10ec255740485dd39d523b4af729196a87c4019c71b0c936c0ab1a50c06 +size 272789 diff --git a/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_32.JPG b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_32.JPG new file mode 100644 index 0000000000000000000000000000000000000000..5fbbac43f4b9f1db09b4fbb27c80a089ee18a4c4 Binary files /dev/null and b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_32.JPG differ diff --git a/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_62.JPG b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_62.JPG new file mode 100644 index 0000000000000000000000000000000000000000..0eb31aecb0721091289660acc91828e437b17d65 --- /dev/null +++ b/Testsets/LR_Whole/ip11pro_output_textSR_211206_renew_62.JPG @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:577f6e43c09ebe078531081914f3f8702144e116bc91ff75b6897344b34e56a4 +size 387052 diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4e0c5013a4908215356d60ad6f798c3c59b1c704 --- /dev/null +++ b/app.py @@ -0,0 +1,117 @@ +import torch +import cv2 +import numpy as np +import os +import os.path as osp +import time +import gradio as gr + +os.environ["GRADIO_TEMP_DIR"] = "./gradio_tmp" + +from models.TextEnhancement import MARCONetPlus +from utils.utils_image import imread_uint, uint2tensor4, tensor2uint +from networks.rrdbnet2_arch import RRDBNet as BSRGAN + +# Initialize device +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + +# Background restoration model (lazy loading) +BGModel = None +def load_bg_model(): + """Load BSRGAN model for background super-resolution""" + global BGModel + if BGModel is None: + BGModel = BSRGAN(in_nc=3, out_nc=3, nf=64, nb=23, gc=32, sf=2) + model_old = torch.load('./checkpoints/bsrgan_bg.pth', map_location=device) + state_dict = BGModel.state_dict() + for ((key, param), (key2, _)) in zip(model_old.items(), state_dict.items()): + state_dict[key2] = param + BGModel.load_state_dict(state_dict, strict=True) + BGModel.eval() + for k, v in BGModel.named_parameters(): + v.requires_grad = False + BGModel = BGModel.to(device) + +# Text restoration model +TextModel = MARCONetPlus( + './checkpoints/net_w_encoder_860000.pth', + './checkpoints/net_prior_860000.pth', + './checkpoints/net_sr_860000.pth', + './checkpoints/yolo11m_short_character.pt', + device=device +) + +def gradio_inference(input_img, aligned=False, bg_sr=False, scale_factor=2): + """Run MARCONetPlus inference with optional background SR""" + if input_img is None: + return None + + # Convert input image (PIL) to OpenCV format + img_L = cv2.cvtColor(np.array(input_img), cv2.COLOR_RGB2BGR) + height_L, width_L = img_L.shape[:2] + + # Background super-resolution + if not aligned and bg_sr: + load_bg_model() + img_E = cv2.resize(img_L, (int(width_L//8*8), int(height_L//8*8)), interpolation=cv2.INTER_AREA) + img_E = uint2tensor4(img_E).to(device) + with torch.no_grad(): + try: + img_E = BGModel(img_E) + except: + torch.cuda.empty_cache() + max_size = 1536 + scale = min(max_size / width_L, max_size / height_L, 1.0) + new_width = int(width_L * scale) + new_height = int(height_L * scale) + img_E = cv2.resize(img_L, (new_width//8*8, new_height//8*8), interpolation=cv2.INTER_AREA) + img_E = uint2tensor4(img_E).to(device) + img_E = BGModel(img_E) + img_E = tensor2uint(img_E) + else: + img_E = img_L + + # Resize background + width_S = width_L * scale_factor + height_S = height_L * scale_factor + img_E = cv2.resize(img_E, (width_S, height_S), interpolation=cv2.INTER_AREA) + + # Text restoration + SQ, ori_texts, en_texts, debug_texts, pred_texts = TextModel.handle_texts( + img=img_L, bg=img_E, sf=scale_factor, is_aligned=aligned + ) + + if SQ is None: + return None + + if not aligned: + SQ = cv2.resize(SQ.astype(np.float32), (width_S, height_S), interpolation=cv2.INTER_AREA) + out_img = SQ[:, :, ::-1].astype(np.uint8) + else: + out_img = en_texts[0][:, :, ::-1].astype(np.uint8) + + return out_img + +# Gradio UI +with gr.Blocks() as demo: + gr.Markdown("# MARCONetPlus Text Image Restoration") + + with gr.Row(): + input_img = gr.Image(type="pil", label="Input Image") + output_img = gr.Image(type="numpy", label="Restored Output") + + with gr.Row(): + aligned = gr.Checkbox(label="Aligned (cropped text regions)", value=False) + bg_sr = gr.Checkbox(label="Background SR (BSRGAN)", value=False) + scale_factor = gr.Slider(1, 4, value=2, step=1, label="Scale Factor") + + run_btn = gr.Button("Run Inference") + + run_btn.click( + fn=gradio_inference, + inputs=[input_img, aligned, bg_sr, scale_factor], + outputs=[output_img] + ) + +if __name__ == "__main__": + demo.launch(server_name="0.0.0.0", server_port=7121) diff --git a/checkpoints/bsrgan_bg.pth b/checkpoints/bsrgan_bg.pth new file mode 100644 index 0000000000000000000000000000000000000000..28232caeafbc633c48f609ff6f926a85d50670fa --- /dev/null +++ b/checkpoints/bsrgan_bg.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82df5fa543c716933bb26592519848c74ada4c86f101324ecb8e9af863c61793 +size 66894171 diff --git a/checkpoints/modelscope_ocr/.lock/damo___cv_convnextTiny_ocr-recognition-general_damo b/checkpoints/modelscope_ocr/.lock/damo___cv_convnextTiny_ocr-recognition-general_damo new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mdl b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mdl new file mode 100644 index 0000000000000000000000000000000000000000..8aa445f3d0e852e2468c8fd21f2c03b0139abd71 Binary files /dev/null and b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mdl differ diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.msc b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.msc new file mode 100644 index 0000000000000000000000000000000000000000..6e9499c3268ba35d6993da3bd4361dfa470d750f Binary files /dev/null and b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.msc differ diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mv b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mv new file mode 100644 index 0000000000000000000000000000000000000000..364dea1a2b166d0730fc1028c704c2403d198408 --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/.mv @@ -0,0 +1 @@ +Revision:v2.4.0,CreatedAt:1695105029 \ No newline at end of file diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/README.md b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f36f6527925d147ecfa2bfe382f5a8ccc1d85769 --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/README.md @@ -0,0 +1,245 @@ +--- +backbone: +- convNext-Tiny +integrating: True +domain: +- cv +frameworks: +- pytorch +language: +- en +- ch +license: Apache License 2.0 +metrics: +- Line Accuracy +finetune-support: True +tags: +- OCR +- Alibaba +- 文字识别 +- 读光 +tasks: +- ocr-recognition + +studios: +- damo/cv_ocr-text-spotting + +datasets: + test: + - damo/WebText_Dataset + +widgets: + - task: ocr-recognition + inputs: + - type: image + examples: + - name: 1 + inputs: + - name: image + data: http://duguang-labelling.oss-cn-shanghai.aliyuncs.com/mass_img_tmp_20220922/ocr_recognition.jpg + +--- + +# 读光文字识别 +## News +- 2023年6月： + - 新增轻量化端侧识别[LightweightEdge-通用场景](https://www.modelscope.cn/models/damo/cv_LightweightEdge_ocr-recognitoin-general_damo/summary)模型和轻量化端侧[行检测模型](https://www.modelscope.cn/models/damo/cv_proxylessnas_ocr-detection-db-line-level_damo/summary)。 +- 2023年4月： + - 新增训练/微调时读取本地数据集的lmdb，用训练/微调后的模型继续识别，详见代码示例。 +- 2023年3月： + - 新增训练/微调流程，支持自定义参数及数据集，详见代码示例。 +- 2023年2月： + - 新增业界主流[CRNN-通用场景](https://www.modelscope.cn/models/damo/cv_crnn_ocr-recognition-general_damo/summary)模型。 + +## 传送门 +各场景文本识别模型： +- [ConvNextViT-手写场景](https://www.modelscope.cn/models/damo/cv_convnextTiny_ocr-recognition-handwritten_damo/summary) +- [ConvNextViT-文档印刷场景](https://www.modelscope.cn/models/damo/cv_convnextTiny_ocr-recognition-document_damo/summary) +- [ConvNextViT-自然场景](https://www.modelscope.cn/models/damo/cv_convnextTiny_ocr-recognition-scene_damo/summary) +- [ConvNextViT-车牌场景](https://www.modelscope.cn/models/damo/cv_convnextTiny_ocr-recognition-licenseplate_damo/summary) +- [CRNN-通用场景](https://www.modelscope.cn/models/damo/cv_crnn_ocr-recognition-general_damo/summary) +- [LightweightEdge-通用场景](https://www.modelscope.cn/models/damo/cv_LightweightEdge_ocr-recognitoin-general_damo/summary) + +各场景文本检测模型： +- [SegLink++-通用场景行检测](https://modelscope.cn/models/damo/cv_resnet18_ocr-detection-line-level_damo/summary) +- [SegLink++-通用场景单词检测](https://modelscope.cn/models/damo/cv_resnet18_ocr-detection-word-level_damo/summary) +- [DBNet-通用场景行检测](https://www.modelscope.cn/models/damo/cv_resnet18_ocr-detection-db-line-level_damo/summary) + +整图OCR能力： +- [整图OCR-多场景](https://modelscope.cn/studios/damo/cv_ocr-text-spotting/summary) + +欢迎使用！ + +## 模型描述 +- 文字识别，即给定一张文本图片，识别出图中所含文字并输出对应字符串。 +- 本模型主要包括三个主要部分，Convolutional Backbone提取图像视觉特征，ConvTransformer Blocks用于对视觉特征进行上下文建模，最后连接CTC loss进行识别解码以及网络梯度优化。识别模型结构如下图： + +

+ +

+ +## 期望模型使用方式以及适用范围 +本模型主要用于给输入图片输出图中文字内容，具体地，模型输出内容以字符串形式输出。用户可以自行尝试各种输入图片。具体调用方式请参考代码示例。 +- 注：输入图片应为包含文字的单行文本图片。其它如多行文本图片、非文本图片等可能没有返回结果，此时表示模型的识别结果为空。 + +## 模型推理 +在安装完成ModelScope之后即可使用ocr-recognition的能力。(在notebook的CPU环境或GPU环境均可使用) +- 使用图像的url，或准备图像文件上传至notebook（可拖拽）。 +- 输入下列代码。 + +### 代码范例 +```python +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +import cv2 + +ocr_recognition = pipeline(Tasks.ocr_recognition, model='damo/cv_convnextTiny_ocr-recognition-general_damo') + +### 使用url +img_url = 'http://duguang-labelling.oss-cn-shanghai.aliyuncs.com/mass_img_tmp_20220922/ocr_recognition.jpg' +result = ocr_recognition(img_url) +print(result) + +### 使用图像文件 +### 请准备好名为'ocr_recognition.jpg'的图像文件 +# img_path = 'ocr_recognition.jpg' +# img = cv2.imread(img_path) +# result = ocr_recognition(img) +# print(result) +``` + +### 模型可视化效果 +以下为模型的可视化文字识别效果。 + +

+ +

+ +### 模型局限性以及可能的偏差 +- 模型是在中英文数据集上训练的，在其他语言的数据上有可能产生一定偏差，请用户自行评测后决定如何使用。 +- 当前版本在python3.7的CPU环境和单GPU环境测试通过，其他环境下可用性待测试。 + +## 模型微调/训练 +### 训练数据及流程介绍 +- 本文字识别模型训练数据集是MTWI以及部分收集数据，训练数据数量约6M。 +- 本模型参数随机初始化，然后在训练数据集上进行训练，在32x300尺度下训练20个epoch。 + +### 模型微调/训练示例 +#### 训练数据集准备 +示例采用[ICDAR13手写数据集](https://modelscope.cn/datasets/damo/ICDAR13_HCTR_Dataset/summary)，已制作成lmdb，数据格式如下 +``` +'num-samples': number, +'image-000000001': imagedata, +'label-000000001': string, +... +``` +详情可下载解析了解。 + +#### 配置训练参数并进行微调/训练 +参考代码及详细说明如下 +```python +import os +import tempfile + +from modelscope.hub.snapshot_download import snapshot_download +from modelscope.metainfo import Trainers +from modelscope.msdatasets import MsDataset +from modelscope.trainers import build_trainer +from modelscope.utils.config import Config, ConfigDict +from modelscope.utils.constant import ModelFile, DownloadMode + +### 请确认您当前的modelscope版本，训练/微调流程在modelscope==1.4.0及以上版本中 +### 当前notebook中版本为1.3.2，请手动更新，建议使用GPU环境 + +model_id = 'damo/cv_convnextTiny_ocr-recognition-general_damo' +cache_path = snapshot_download(model_id) # 模型下载保存目录 +config_path = os.path.join(cache_path, ModelFile.CONFIGURATION) # 模型参数配置文件，支持自定义 +cfg = Config.from_file(config_path) + +# 构建数据集，支持自定义 +train_data_cfg = ConfigDict( + name='ICDAR13_HCTR_Dataset', + split='test', + namespace='damo', + test_mode=False) + +train_dataset = MsDataset.load( + dataset_name=train_data_cfg.name, + split=train_data_cfg.split, + namespace=train_data_cfg.namespace, + download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) + +test_data_cfg = ConfigDict( + name='ICDAR13_HCTR_Dataset', + split='test', + namespace='damo', + test_mode=True) + +test_dataset = MsDataset.load( + dataset_name=test_data_cfg.name, + split=test_data_cfg.split, + namespace=train_data_cfg.namespace, + download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) + +tmp_dir = tempfile.TemporaryDirectory().name # 模型文件和log保存位置，默认为"work_dir/" + +# 自定义参数，例如这里将max_epochs设置为15，所有参数请参考configuration.json +def _cfg_modify_fn(cfg): + cfg.train.max_epochs = 15 + return cfg + +#################################################################################### + +''' +使用本地文件 + lmdb: + 构建包含下列信息的lmdb文件 (key: value) + 'num-samples': 总样本数, + 'image-000000001': 图像的二进制编码, + 'label-000000001': 标签序列的二进制编码, + ... + image和label后的index为9位并从1开始 +下面为示例 (local_lmdb为本地的lmdb文件) +''' + +# train_dataset = MsDataset.load( +# dataset_name=train_data_cfg.name, +# split=train_data_cfg.split, +# namespace=train_data_cfg.namespace, +# download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS, +# local_lmdb='./local_lmdb') + +# test_dataset = MsDataset.load( +# dataset_name=test_data_cfg.name, +# split=test_data_cfg.split, +# namespace=train_data_cfg.namespace, +# download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS, +# local_lmdb='./local_lmdb') + +#################################################################################### + +kwargs = dict( + model=model_id, + train_dataset=train_dataset, + eval_dataset=test_dataset, + work_dir=tmp_dir, + cfg_modify_fn=_cfg_modify_fn) + +# 模型训练 +trainer = build_trainer(name=Trainers.ocr_recognition, default_args=kwargs) +trainer.train() +``` + +#### 用训练/微调后的模型进行识别 +```python +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +import os + +ep_num = 3 # 选择模型checkpoint +cmd = 'cp {} {}'.format('./work_dir/epoch_%d.pth' % ep_num, './work_dir/output/pytorch_model.pt') # 'work_dir'为configuration中设置的路径，'output'为输出默认路径 +os.system(cmd) +ocr_recognition = pipeline(Tasks.ocr_recognition, model='./work_dir/output' ) +result = ocr_recognition('http://duguang-labelling.oss-cn-shanghai.aliyuncs.com/mass_img_tmp_20220922/ocr_recognition_icdar13.jpg') +print(result) +``` \ No newline at end of file diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/configuration.json b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/configuration.json new file mode 100644 index 0000000000000000000000000000000000000000..12234e1d3eaeef6593b87a90ef10f7918f69b39f --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/configuration.json @@ -0,0 +1,68 @@ +{ + "framework": "pytorch", + "task": "ocr-recognition", + "pipeline": { + "type": "convnextTiny-ocr-recognition" + }, + "model": { + "type": "OCRRecognition", + "recognizer": "ConvNextViT", + "inference_kwargs": { + "img_height": 32, + "img_width": 804, + "do_chunking": true + } + }, + "preprocessor": { + "type": "ocr-recognition" + }, + "train": { + "max_epochs": 30, + "work_dir": "./work_dir", + "dataloader": { + "batch_size_per_gpu": 32, + "workers_per_gpu": 0 + }, + "optimizer": { + "type": "AdamW", + "weight_decay": 0.01, + "lr": 0.001, + "options": { + "grad_clip": { + "max_norm": 20 + } + } + }, + "lr_scheduler": { + "type": "MultiStepLR", + "milestones": [10, 20], + "gamma": 0.1 + }, + "hooks": [{ + "type": "CheckpointHook", + "interval": 1, + "save_dir": "./work_dir" + }, + { + "type": "TextLoggerHook", + "interval": 50, + "out_dir": "./work_dir" + }, + { + "type": "IterTimerHook" + }, + { + "type": "EvaluationHook", + "interval": 1 + } + ] + }, + "evaluation": { + "dataloader": { + "batch_size_per_gpu": 32, + "workers_per_gpu": 0, + "shuffle": false + }, + "metrics": "ocr-recognition-metric" + } +} \ No newline at end of file diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/model.onnx b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e85f0ec6ea719c79a8d3e97f549da438a588f080 --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b4a624de1f099fede53622cf63e40fc07e2f09f1ef4e39d62bbc44858efdef4 +size 76859029 diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/pytorch_model.pt b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/pytorch_model.pt new file mode 100644 index 0000000000000000000000000000000000000000..803125e6b39791c4f72a2aaf06b72fd41f81e199 --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/pytorch_model.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31cc4d17982aa82cfc7acfb95553b414183eaafafc7ff199692d4ebfbed7243c +size 76836856 diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/quickstart.md b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/quickstart.md new file mode 100644 index 0000000000000000000000000000000000000000..5d6f8a9a12a28031d4fd0279957b2afeb00b1cb0 --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/quickstart.md @@ -0,0 +1,161 @@ +--- + + +--- +## 模型推理 +- 使用图像的url，或准备图像文件。 +- 输入下列代码。 + +### 代码范例 +```python +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +import cv2 + +ocr_recognition = pipeline(Tasks.ocr_recognition, model='damo/cv_convnextTiny_ocr-recognition-general_damo') + +### 使用url +img_url = 'http://duguang-labelling.oss-cn-shanghai.aliyuncs.com/mass_img_tmp_20220922/ocr_recognition.jpg' +result = ocr_recognition(img_url) +print(result) + +### 使用图像文件 +### 请准备好名为'ocr_recognition.jpg'的图像文件 +# img_path = 'ocr_recognition.jpg' +# img = cv2.imread(img_path) +# result = ocr_recognition(img) +# print(result) +``` +更多关于模型加载和推理的问题参考[模型的推理Pipeline](https://modelscope.cn/docs/%E6%A8%A1%E5%9E%8B%E7%9A%84%E6%8E%A8%E7%90%86Pipeline)。 + +## 模型训练 +### 模型微调/训练示例 +#### 训练数据集准备 +示例采用[ICDAR13手写数据集](https://modelscope.cn/datasets/damo/ICDAR13_HCTR_Dataset/summary)，已制作成lmdb，数据格式如下 +``` +'num-samples': number, +'image-000000001': imagedata, +'label-000000001': string, +... +``` +详情可下载解析了解。 + +#### 配置训练参数并进行微调/训练 +参考代码及详细说明如下 +```python +import os +import tempfile + +from modelscope.hub.snapshot_download import snapshot_download +from modelscope.metainfo import Trainers +from modelscope.msdatasets import MsDataset +from modelscope.trainers import build_trainer +from modelscope.utils.config import Config, ConfigDict +from modelscope.utils.constant import ModelFile, DownloadMode + +### 请确认您当前的modelscope版本，训练/微调流程在modelscope==1.4.0及以上版本中 + +model_id = 'damo/cv_convnextTiny_ocr-recognition-general_damo' +cache_path = snapshot_download(model_id) # 模型下载保存目录 +config_path = os.path.join(cache_path, ModelFile.CONFIGURATION) # 模型参数配置文件，支持自定义 +cfg = Config.from_file(config_path) + +# 构建数据集，支持自定义 +train_data_cfg = ConfigDict( + name='ICDAR13_HCTR_Dataset', + split='test', + namespace='damo', + test_mode=False) + +train_dataset = MsDataset.load( + dataset_name=train_data_cfg.name, + split=train_data_cfg.split, + namespace=train_data_cfg.namespace, + download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) + +test_data_cfg = ConfigDict( + name='ICDAR13_HCTR_Dataset', + split='test', + namespace='damo', + test_mode=True) + +test_dataset = MsDataset.load( + dataset_name=test_data_cfg.name, + split=test_data_cfg.split, + namespace=train_data_cfg.namespace, + download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS) + +tmp_dir = tempfile.TemporaryDirectory().name # 模型文件和log保存位置，默认为"work_dir/" + +# 自定义参数，例如这里将max_epochs设置为15，所有参数请参考configuration.json +def _cfg_modify_fn(cfg): + cfg.train.max_epochs = 15 + return cfg + +#################################################################################### + +''' +使用本地文件 + lmdb: + 构建包含下列信息的lmdb文件 (key: value) + 'num-samples': 总样本数, + 'image-000000001': 图像的二进制编码, + 'label-000000001': 标签序列的二进制编码, + ... + image和label后的index为9位并从1开始 +下面为示例 (local_lmdb为本地的lmdb文件) +''' + +# train_dataset = MsDataset.load( +# dataset_name=train_data_cfg.name, +# split=train_data_cfg.split, +# namespace=train_data_cfg.namespace, +# download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS, +# local_lmdb='./local_lmdb') + +# test_dataset = MsDataset.load( +# dataset_name=test_data_cfg.name, +# split=test_data_cfg.split, +# namespace=train_data_cfg.namespace, +# download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS, +# local_lmdb='./local_lmdb') + +#################################################################################### + +kwargs = dict( + model=model_id, + train_dataset=train_dataset, + eval_dataset=test_dataset, + work_dir=tmp_dir, + cfg_modify_fn=_cfg_modify_fn) + +# 模型训练 +trainer = build_trainer(name=Trainers.ocr_recognition, default_args=kwargs) +trainer.train() +``` + +#### 用训练/微调后的模型进行识别 +```python +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +import os + +ep_num = 3 # 选择模型checkpoint +cmd = 'cp {} {}'.format('./work_dir/epoch_%d.pth' % ep_num, './work_dir/output/pytorch_model.pt') # 'work_dir'为configuration中设置的路径，'output'为输出默认路径 +os.system(cmd) +ocr_recognition = pipeline(Tasks.ocr_recognition, model='./work_dir/output' ) +result = ocr_recognition('http://duguang-labelling.oss-cn-shanghai.aliyuncs.com/mass_img_tmp_20220922/ocr_recognition_icdar13.jpg') +print(result) +``` +更多使用说明请参阅[ModelScope文档中心](http://www.modelscope.cn/#/docs)。 + +--- + + +--- +## 下载并安装ModelScope library +更多关于下载安装ModelScope library的问题参考[环境安装](https://modelscope.cn/docs/%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85)。 + +```python +pip install "modelscope[cv]" -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +``` diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/ConvTransformer-Pipeline.jpg b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/ConvTransformer-Pipeline.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7587bd699b99151ef4ec9cda62042d17218a3eeb --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/ConvTransformer-Pipeline.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc64960ec8dd7e821461256c8170e7db41e31287a6d6f507591d5557440126c9 +size 102782 diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_measure.png b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_measure.png new file mode 100644 index 0000000000000000000000000000000000000000..e923cab4b4e94d66fee14b6ca3be99fc61d2bbc1 Binary files /dev/null and b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_measure.png differ diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_visu.jpg b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_visu.jpg new file mode 100644 index 0000000000000000000000000000000000000000..47d5ead3c3a20708931fffcf8a8d3b4c6e5a835b Binary files /dev/null and b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/resources/rec_result_visu.jpg differ diff --git a/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/vocab.txt b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..45c2874900eb39b62073fd2489ca9b793e937adf --- /dev/null +++ b/checkpoints/modelscope_ocr/damo/cv_convnextTiny_ocr-recognition-general_damo/vocab.txt @@ -0,0 +1,7643 @@ +在 +保 +证 +商 +品 +和 +外 +包 +装 +完 +好 +无 +损 +情 +况 +下 +与 +本 +店 +客 +服 +联 +系 +并 +百 +代 +采 +购 +周 +期 +为 +1 +- +3 +天 +二 +检 +测 +了 +提 +香 +。 +单 +项 +深 +度 +法 +的 +目 +标 +艺 +术 +对 +其 +他 +发 +展 +领 +域 +促 +进 +五 +虚 +伪 +人 +味 +及 +雅 +量 +第 +十 +章 +袁 +世 +凯 +你 +何 +成 +不 +华 +盛 +顿 +一 +精 +塑 +封 +因 +此 +可 +以 +得 +出 +两 +个 +结 +论 +是 +大 +道 +养 +生 +中 +医 +西 +都 +拒 +绝 +没 +有 +能 +具 +备 +力 +朵 +莲 +花 +按 +照 +湿 +地 +态 +统 +更 +多 +新 +款 +欢 +迎 +请 +加 +主 +林 +动 +眼 +神 +阴 +沉 +望 +着 +那 +夹 +杂 +狂 +暴 +波 +掠 +而 +来 +元 +光 +芒 +电 +子 +产 +较 +手 +笔 +京 +城 +饭 +顶 +层 +旋 +转 +餐 +厅 +绞 +股 +蓝 +抗 +衰 +老 +、 +预 +防 +肿 +瘤 +抑 +制 +溃 +疡 +缓 +解 +紧 +张 +疲 +劳 +镇 +蛋 +糕 +配 +套 +赠 +椰 +滚 +落 +水 +时 +又 +恰 +巧 +砸 +晕 +正 +旁 +边 +看 +热 +闹 +鱼 +陈 +荣 +茂 +帮 +年 +轻 +朋 +友 +朝 +气 +蓬 +勃 +善 +于 +学 +习 +勇 +创 +顾 +阑 +珊 +刚 +走 +门 +7 +. +2 +设 +计 +药 +形 +局 +限 +性 +社 +区 +心 +理 +康 +复 +家 +庭 +七 +我 +们 +否 +应 +该 +停 +忙 +碌 +脚 +步 +唉 +它 +说 +过 +朴 +素 +【 +介 +绍 +】 +刑 +公 +感 +( +江 +苏 +) +买 +日 +0 +6 +8 +挺 +P +r +e +n +a +t +l +N +u +i +o +d +f +s +几 +乎 +被 +噩 +运 +溺 +毙 +她 +后 +化 +作 +李 +源 +儿 +再 +回 +到 +非 +之 +付 +红 +屁 +疹 +4 +内 +容 +识 +别 +带 +风 +扇 +机 +纯 +铜 +高 +梁 +强 +措 +施 +合 +安 +排 +受 +方 +式 +书 +布 +朗 +小 +《 +使 +魔 +鬼 +》 +A +g +研 +究 +三 +节 +独 +山 +石 +分 +司 +全 +面 +算 +管 +用 +实 +例 +四 +国 +际 +贸 +易 +划 +程 +骤 +类 +型 +这 +散 +文 +概 +念 +# +S +w +p +C +m +钢 +膜 +屏 +覆 +盖 +炉 +铁 +口 +液 +压 +开 +钻 +杆 +自 +注 +浆 +锚 +体 +9 +5 +, +h +c +v +B +初 +次 +或 +长 +放 +置 +定 +要 +充 +会 +查 +验 +库 +存 +通 +知 +您 +货 +; +月 +O +海 +拔 +: +均 +超 +临 +床 +必 +读 +丛 +厘 +摩 +Y +G +R +毫 +克 +镁 +码 +关 +九 +官 +选 +士 +举 +土 +工 +立 +纹 +饰 +从 +群 +治 +疗 +农 +业 +技 +校 +讲 +述 +上 +质 +错 +邮 +浙 +川 +贵 +州 +云 +南 +藏 +也 +叫 +邦 +义 +军 +政 +重 +确 +牛 +皮 +癣 +症 +经 +组 +减 +观 +剧 +须 +戏 +捧 +教 +版 +课 +准 +依 +据 +俄 +普 +希 +金 +| +编 +燕 +译 +者 +吕 +捷 +绘 +画 +宏 +亚 +图 +双 +显 +卡 +智 +切 +换 +偿 +乙 +肝 +炎 +硬 +患 +还 +啦 +简 +已 +车 +胶 +清 +洁 +位 +直 +接 +黏 +即 +号 +适 +哦 +纸 +I +D +瑞 +丽 +志 +德 +伯 +特 +灵 +格 +蔡 +部 +温 +控 +器 +优 +势 +吴 +景 +横 +晖 +郭 +玉 +费 +卖 +各 +承 +担 +半 +假 +若 +键 +刻 +醒 +每 +种 +只 +室 +入 +处 +事 +美 +恩 +房 +龙 +著 +晏 +榕 +莫 +尔 +杜 +霍 +维 +奇 +英 +怀 +杨 +富 +斌 +最 +修 +订 +权 +威 +登 +舞 +银 +牌 +级 +* +y +z +V +/ +H +J +k +专 +样 +前 +试 +剂 +盒 +府 +奖 +持 +宝 +肤 +常 +健 +基 +础 +指 +短 +线 +操 +旅 +游 +纪 +币 +间 +价 +值 +涵 +环 +境 +审 +物 +畅 +销 +影 +响 +启 +蒙 +虽 +然 +答 +拉 +里 +料 +故 +誓 +浪 +漫 +燃 +烧 +脏 +平 +随 +呈 +增 +太 +阳 +察 +护 +T +M +尺 +表 +某 +件 +衣 +寸 +参 +考 +六 +角 +螺 +栓 +固 +盗 +给 +籍 +借 +钱 +破 +坏 +融 +秩 +序 +罪 +律 +责 +任 +民 +院 +反 +倾 +行 +案 +干 +问 +题 +规 ++ +植 +绒 +背 +板 +b +需 +向 +咨 +陶 +瓷 +球 +油 +润 +滑 +属 +擦 +数 +L +韩 +谨 +骗 +推 +荐 +喜 +洋 +底 +状 +让 +延 +握 +柄 +滤 +雾 +℃ +氧 +火 +原 +杏 +仁 +芝 +迷 +E +埃 +汇 +韵 +达 +等 +快 +递 +如 +添 +珍 +惜 +藻 +透 +明 +W +? += +拍 +岩 +鲜 +汀 +永 +杭 +微 +~ +K +速 +浏 +览 +F +千 +米 +示 +宽 +比 +余 +条 +务 +语 +秒 +连 +续 +信 +息 +U +堂 +般 +蚝 +鸡 +浅 +折 +叠 +虞 +劲 +松 +拐 +擎 +杰 +斐 +逊 +x +黄 +暗 +蒋 +猜 +谜 +宣 +言 +交 +锋 +词 +省 +略 +果 +泥 +渣 +砖 +砌 +块 +造 +妇 +罗 +满 +佳 +八 +鸿 +運 +木 +材 +演 +經 +典 +風 +刮 +壁 +星 +搅 +拌 +乳 +卷 +晋 +胡 +萝 +卜 +饼 +哭 +吧 +j +将 +芯 +哪 +些 +贝 +惠 +券 +命 +名 +则 +密 +佛 +珠 +串 +% +隔 +兰 +引 +孩 +铝 +架 +男 +女 +古 +鲁 +迅 +逆 +耳 +话 +忠 +效 +办 +肠 +便 +秘 +孕 +妈 +幽 +默 +打 +扉 +阅 +册 +循 +渐 +字 +喻 +贫 +贡 +献 +界 +移 +现 +挑 +战 +流 +网 +络 +误 +功 +点 +认 +色 +彩 +磁 +路 +构 +矢 +场 +泄 +泻 +园 +建 +变 +频 +冻 +桥 +震 +害 +企 +含 +符 +Q +片 +污 +染 +析 +模 +拟 +X +集 +供 +链 +粒 +袋 +相 +耕 +械 +趋 +活 +资 +市 +吞 +吐 +投 +险 +访 +同 +济 +仲 +裁 +诉 +讼 +核 +极 +晶 +输 +曲 +获 +取 +助 +监 +由 +团 +户 +账 +弧 +附 +帐 +汗 +调 +慢 +跳 +拦 +综 +征 +矫 +函 +箱 +血 +糖 +仪 +做 +细 +径 +档 +先 +声 +音 +冷 +凝 +称 +当 +才 +支 +赔 +寄 +磕 +碰 +难 +免 +掉 +漆 +敬 +见 +毒 +夜 +雨 +避 +缝 +吸 +盘 +议 +坐 +姿 +所 +售 +翡 +翠 +镯 +缅 +甸 +改 +织 +割 +截 +圆 +峰 +就 +沪 +首 +斤 +起 +墨 +粘 +斯 +矛 +盾 +意 +盟 +玻 +璃 +汁 +瓶 +洗 +忧 +笑 +赛 +乃 +姆 +灌 +缩 +衷 +季 +酸 +酵 +氨 +纶 +聚 +酯 +纤 +差 +空 +伸 +弯 +辑 +估 +乐 +票 +椭 +令 +拓 +训 +练 +倪 +思 +博 +传 +策 +把 +己 +茶 +段 +冲 +醚 +妙 +填 +趣 +@ +妆 +沐 +浴 +露 +吊 +Z +燈 +圓 +伞 +狐 +狸 +毛 +断 +降 +恢 +亮 +颗 +离 +孔 +钟 +跑 +消 +耗 +缘 +阻 +吉 +虎 +鼠 +共 +鹰 +锈 +柱 +蜗 +亲 +索 +求 +评 +凭 +广 +告 +墅 +焊 +悬 +臂 +整 +堆 +未 +梳 +摄 +菌 +菇 +棚 +夏 +印 +卫 +批 +候 +送 +拼 +硼 +税 +视 +暖 +轴 +镜 +执 +粗 +― +丈 +夫 +职 +田 +纳 +慕 +宁 +静 +万 +枚 +饿 +蛇 +桢 +鸵 +鸟 +栗 +桂 +炖 +猪 +蹄 +[ +] +扰 +棒 +稳 +零 +师 +葱 +什 +么 +退 +协 +利 +真 +遗 +头 +记 +脑 +升 +互 +溫 +莎 +爵 +愛 +象 +徵 +很 +瀑 +怎 +骨 +击 +奏 +弘 +師 +開 +剑 +休 +颜 +耐 +蕴 +铀 +列 +枕 +醋 +敷 +雷 +萨 +轿 +疑 +障 +汽 +缸 +棕 +浦 +伦 +敦 +食 +伤 +身 +圣 +迹 +稀 +青 +巅 +雄 +潤 +灯 +' +白 +豹 +桶 +另 +营 +率 +爆 +绳 +扣 +丸 +睡 +遣 +削 +弱 +异 +壮 +收 +颈 +椎 +病 +吗 +酱 +根 +宗 +盐 +焗 +筒 +晓 +珑 +辽 +兴 +扩 +幸 +福 +蒲 +听 +馆 +祭 +北 +谓 +臀 +肌 +挛 +归 +猴 +脱 +磷 +诺 +东 +县 +今 +邹 +村 +历 +涯 +副 +席 +央 +擅 +兽 +毕 +武 +汉 +科 +祝 +范 +泰 +斗 +哲 +兼 +闻 +许 +留 +至 +乌 +党 +派 +坪 +寒 +厂 +铃 +锐 +港 +航 +触 +座 +众 +钮 +尽 +致 +慎 +申 +丹 +欧 +筑 +赋 +媒 +低 +炮 +草 +橡 +顺 +丰 +总 +委 +决 +仿 +辅 +射 +钳 +锯 +锉 +去 +幕 +菜 +择 +软 +刀 +牙 +王 +终 +炒 +锅 +磨 +树 +脂 +浸 +阿 +巴 +舍 +抹 +灰 +录 +矩 +坝 +咖 +仓 +额 +紫 +磅 +积 +近 +似 +龄 +丝 +池 +颠 +谐 +谱 +阶 +报 +约 +偶 +写 +赏 +轨 +掺 +搞 +残 +导 +溶 +铅 +搭 +员 +契 +站 +围 +苯 +胺 +址 +尼 +谈 +判 +纵 +描 +像 +塞 +楼 +荷 +载 +锁 +烈 +诱 +欲 +垫 +骑 +寿 +失 +婚 +姻 +继 +诚 +腊 +庙 +予 +摇 +篮 +浮 +廉 +良 +宿 +句 +逻 +沟 +困 +涉 +询 +爱 +革 +找 +突 +嘉 +庆 +痛 +圳 +届 +洲 +足 +锦 +旗 +住 +惑 +秋 +冬 +枪 +萌 +闪 +熊 +猫 +衫 +杯 +伟 +纱 +童 +始 +刷 +雪 +纺 +厚 +针 +潮 +撞 +尚 +春 +棉 +麻 +恤 +侣 +薄 +拖 +鞋 +凉 +津 +肩 +闲 +羊 +巾 +黑 +胖 +裤 +腰 +肥 +袖 +衬 +裙 +豆 +胸 +翻 +唯 +舰 +框 +偏 +脸 +跟 +台 +嘴 +镂 +巡 +唱 +蝴 +蝶 +仔 +帽 +斜 +挎 +瘦 +熟 +轮 +煎 +艾 +蒿 +粑 +岁 +扭 +荧 +早 +享 +育 +苗 +马 +丁 +括 +遵 +守 +除 +疆 +讨 +兑 +涤 +钓 +椅 +渔 +竿 +兄 +弟 +煤 +跨 +屋 +檩 +恋 +仅 +莱 +勒 +泵 +寬 +長 +燥 +洞 +穴 +益 +靠 +父 +铺 +储 +倒 +株 +沿 +贴 +招 +剩 +剪 +凡 +霜 +梨 +涛 +页 +尤 +羽 +枣 +爬 +尾 +钠 +钾 +蒸 +镀 +宰 +辨 +娃 +哈 +窟 +少 +追 +疸 +焦 +胆 +母 +玩 +壳 +纲 +末 +驱 +衡 +殊 +垮 +栈 +嵌 +粉 +碎 +租 +赁 +贷 +烟 +囱 +族 +绩 +涡 +垂 +隐 +俗 +左 +右 +篇 +陌 +讪 +拿 +颖 +绿 +酒 +犯 +既 +遂 +队 +胃 +厕 +敌 +忆 +坤 +培 +蜚 +絮 +飘 +萍 +懒 +柴 +董 +昌 +昼 +摊 +居 +携 +郑 +爽 +狗 +漏 +束 +脊 +补 +齐 +端 +窥 +往 +① +叶 +琴 +捕 +鲫 +腐 +蚀 +凹 +坑 +裂 +缺 +陷 +混 +挽 +觉 +阔 +想 +钩 +「 +」 +癫 +痫 +释 +妖 +怪 +氏 +! +插 +杀 +卉 +争 +恶 +剖 +阀 +帜 +探 +圈 +摸 +坡 +份 +崎 +飞 +锤 +歌 +拧 +懂 +沈 +忽 +悠 +钉 +闭 +晚 +播 +菩 +愿 +锂 +佬 +吹 +貌 +骄 +傲 +麦 +蒂 +幻 +裹 +侦 +脖 +驾 +楠 +柜 +_ +脉 +寻 +托 +宜 +撕 +恨 +脆 +绚 +禁 +忌 +绣 +穿 +痕 +浓 +熔 +详 +碳 +酶 +滨 +仙 +途 +铣 +蹈 +淘 +隧 +饱 +疏 +勾 +阵 +瞻 +远 +瞩 +诀 +攻 +踪 +灾 +森 +允 +越 +炼 +距 +竞 +喀 +旭 +寨 +玫 +瑰 +烤 +礼 +肉 +叹 +拆 +砝 +野 +炸 +驴 +慰 +船 +勘 +井 +够 +勿 +掌 +泽 +碑 +仕 +勺 +奶 +虹 +抱 +午 +罐 +渴 +抵 +弃 +负 +遇 +贯 +窗 +债 +哥 +虫 +氢 +榜 +赢 +矿 +河 +噪 +漂 +拥 +宇 +宙 +髓 +柳 +菟 +桃 +芽 +旬 +践 +弹 +趾 +梦 +镶 +抓 +眨 +睛 +隙 +久 +咱 +淋 +锌 +帧 +糊 +挤 +诊 +辈 +淀 +催 +帘 +蚊 +挂 +妻 +矮 +姓 +奕 +卧 +刃 +侧 +抉 +硅 +努 +舒 +铍 +急 +疾 +蜂 +蜇 +齿 +酰 +碱 +瓣 +救 +冗 +槽 +讯 +侵 +占 +财 +绕 +啤 +虑 +烘 +焙 +魅 +★ +竹 +捏 +惯 +靶 +尿 +纫 +沥 +遥 +穷 +甲 +烯 +卒 +危 +敏 +褪 +坠 +铂 +腭 +戒 +缠 +绵 +恒 +耀 +鲤 +警 +泡 +節 +孫 +寶 +琦 +續 +與 +談 +吃 +抬 +栅 +溢 +塔 +戴 +坚 +抽 +袜 +皿 +胁 +且 +钞 +帅 +碧 +柔 +砂 +膏 +q +豪 +蝙 +蝠 +佩 +醫 +醉 +淡 +诗 +櫻 +曼 +頓 +诞 +栽 +眉 +檀 +桑 +柑 +橘 +晨 +Ⅳ +授 +绽 +姐 +胰 +腺 +钛 +摆 +Ⅲ +刺 +乡 +粮 +榨 +筋 +箍 +墙 +禧 +驰 +骋 +烷 +\ +愉 +蔬 +蔻 +犬 +宠 +悦 +吟 +△ +映 +灸 +耍 +$ +洽 +玛 +瑙 +待 +厌 +烦 +橙 +旦 +抄 +袭 +雕 +班 +钙 +肽 +魚 +為 +稱 +對 +體 +類 +呢 +氩 +弦 +私 +街 +叉 +倩 +赵 +谷 +弊 +辐 +肚 +兜 +毯 +屉 +洛 +徒 +奥 +筹 +祈 +振 +朱 +宋 +舆 +胜 +穆 +洙 +严 +佟 +签 +廊 +迈 +戈 +皇 +帝 +菱 +婴 +骆 +驼 +帆 +赶 +碗 +靴 +葩 +嫦 +娥 +虾 +汤 +腾 +渊 +苹 +尖 +時 +盏 +幼 +珂 +吻 +芙 +妮 +陆 +囊 +摔 +膀 +胱 +唇 +硕 +揽 +旧 +{ +} +旺 +※ +Ⅵ +② +趺 +禅 +腿 +③ +④ +弗 +紀 +庫 +顧 +問 +創 +辦 +許 +■ +▲ +▼ +谢 +累 +◆ +坊 +◇ +暂 +○ +◎ +邪 +惬 +● +撼 +萧 +咪 +裱 +☆ +冠 +亏 +唐 +娜 +史 +楚 +眸 +葛 +贤 +劝 +箴 +伺 +啊 +翅 +怨 +叙 +止 +瘢 +挥 +豇 +喷 +泉 +绸 +涩 +乱 +□ +牧 +谁 +咕 +噜 +昏 +焓 +隼 +阐 +姑 +娘 +丫 +鬟 +肢 +腹 +饮 +彼 +潘 +烹 +饪 +烃 +準 +斷 +進 +場 +點 +訣 +竅 +逢 +奴 +赞 +呵 +悉 +浩 +劫 +辟 +冰 +淇 +刊 +抢 +娱 +踏 +夸 +慧 +婉 +腔 +稍 +聘 +鲑 +卵 +峻 +谣 +怠 +歉 +町 +诫 +痰 +郁 +脾 +昧 +棱 +崇 +祯 +却 +晰 +腻 +扬 +麟 +昆 +韭 +椒 +雍 +疯 +粳 +稻 +獵 +繼 +尋 +莉 +悟 +辛 +苦 +兔 +仗 +碼 +丑 +柯 +芳 +苓 +绎 +哀 +枭 +缔 +迄 +颇 +朽 +佶 +撰 +蕾 +療 +術 +務 +陕 +踢 +净 +恐 +慌 +秀 +滔 +讳 +绪 +帕 +刘 +吾 +谚 +眠 +鼾 +死 +姜 +辣 +疼 +後 +愚 +疫 +脐 +翁 +環 +機 +懷 +裡 +芬 +佣 +沙 +漠 +沸 +痹 +澳 +丙 +碟 +靓 +殖 +阈 +褐 +虱 +泪 +搜 +窑 +岸 +耦 +祖 +乘 +沃 +撑 +缴 +筛 +魄 +膨 +胀 +繁 +茎 +贾 +屠 +岳 +拳 +適 +獨 +裝 +設 +計 +會 +來 +聆 +亡 +鸦 +埠 +啡 +诈 +祥 +煞 +倍 +堵 +橱 +伴 +禽 +緻 +欣 +诵 +曾 +皂 +祛 +斑 +跌 +毅 +剃 +僧 +返 +糟 +屈 +寂 +寞 +劣 +癌 +滿 +氛 +圍 +秉 +栋 +襟 +辩 +废 +邀 +泼 +梯 +镍 +踩 +栏 +绊 +迟 +稿 +窍 +卢 +焕 +君 +姚 +嘲 +讽 +畸 +巨 +棋 +梅 +尹 +鹅 +杉 +薛 +玄 +爸 +帖 +亿 +忍 +佚 +谦 +粤 +涂 +纽 +茨 +韦 +剔 +廷 +瑶 +谛 +枸 +杞 +薯 +龟 +羲 +亭 +『 +』 +堇 +現 +貨 +厨 +缚 +娇 +猿 +瓦 +楞 +宫 +彬 +玳 +瑁 +約 +義 +佰 +喇 +叭 +荡 +拘 +窄 +彦 +俱 +忘 +迪 +锆 +馨 +煲 +睿 +岛 +藕 +铬 +遮 +亥 +肖 +湖 +〔 +〕 +扛 +冒 +〓 +甘 +録 +岗 +卤 +坦 +唾 +召 +贽 +纠 +纷 +饲 +筮 +說 +於 +陽 +謂 +凤 +贺 +嫁 +粥 +伙 +肾 +俊 +傻 +徽 +庐 +徐 +泛 +尘 +孽 +慈 +悲 +但 +俯 +恭 +栩 +鹿 +妥 +肯 +悔 +辞 +愁 +扯 +彻 +甩 +泊 +饥 +狼 +箔 +剥 +钴 +败 +垛 +楂 +枝 +甚 +辆 +嗅 +疤 +鉴 +幅 +誉 +孝 +贩 +欠 +庄 +轭 +勤 +呼 +遍 +栖 +泌 +羌 +硝 +盆 +鼻 +葡 +萄 +酷 +炫 +浒 +峡 +寇 +颉 +蓄 +猎 +卸 +極 +數 +總 +邻 +拾 +喂 +鸭 +￥ +昕 +凿 +兵 +妃 +瞧 +哟 +敲 +捆 +亦 +哉 +乏 +霉 +喝 +搓 +將 +罩 +揉 +尴 +尬 +敢 +艰 +萃 +篷 +拎 +稠 +俭 +翼 +岭 +笼 +垃 +圾 +庞 +雀 +嘱 +姬 +铭 +笛 +葫 +芦 +杠 +臭 +贪 +贿 +赂 +滞 +奠 +扳 +皆 +莞 +丢 +衔 +〗 +丧 +励 +逃 +侍 +湘 +霸 +诸 +魂 +傩 +驻 +昊 +湾 +噎 +澡 +弄 +轩 +祸 +迫 +睫 +斓 +壶 +秤 +伍 +搬 +怡 +替 +谋 +嗣 +乔 +胳 +膊 +羞 +炔 +庸 +匡 +乞 +瑕 +疵 +怕 +驭 +珀 +敛 +讓 +戀 +蠢 +陋 +巩 +押 +番 +茄 +狱 +曹 +兹 +寓 +惊 +畜 +葵 +铸 +伽 +瑜 +柬 +埔 +歙 +砚 +魏 +偷 +烛 +馈 +督 +咽 +胎 +酪 +黎 +跃 +蹬 +簧 +瘾 +沂 +贮 +砍 +颌 +鸥 +彈 +詞 +輯 +臣 +孙 +蓉 +赚 +釉 +溪 +泸 +沽 +剿 +捻 +沫 +匀 +巢 +秦 +竣 +矾 +爹 +锡 +溧 +辗 +鼓 +莆 +褥 +奉 +藉 +颅 +糯 +咯 +惧 +黛 +孵 +哺 +刹 +麼 +縮 +們 +沒 +餘 +仆 +劈 +骂 +傅 +傍 +疮 +赖 +耿 +籽 +仍 +殿 +卓 +俞 +滇 +缤 +尧 +沛 +奔 +裸 +尊 +喊 +奋 +赤 +聪 +瞎 +烂 +掩 +揭 +畏 +扎 +涅 +诲 +披 +稚 +盈 +遭 +琢 +孤 +猩 +盔 +菲 +翰 +鲍 +秆 +蔗 +甜 +颤 +抖 +蚱 +蜢 +撒 +蟾 +逾 +泳 +鳖 +拢 +梭 +灿 +娴 +淑 +佑 +荒 +曝 +捣 +捞 +拣 +呆 +堪 +聋 +砭 +挟 +逼 +枫 +挡 +挚 +酣 +漓 +茫 +喵 +菊 +桡 +疝 +璧 +昨 +藤 +蔓 +莓 +伊 +厄 +瓜 +绅 +谅 +祺 +剌 +潜 +毁 +鼎 +砰 +忐 +忑 +饕 +餮 +惆 +怅 +竟 +鞍 +壤 +晃 +闷 +屡 +拜 +舌 +噢 +宅 +缎 +焚 +苍 +煌 +钧 +栾 +彭 +麒 +泓 +筱 +汪 +陵 +鹏 +亨 +御 +瑛 +蕊 +辉 +啥 +吼 +激 +蹭 +岂 +蹋 +砒 +蜜 +汰 +晾 +尝 +嗯 +彰 +掰 +拭 +彝 +嫩 +宛 +枯 +萎 +晒 +淫 +梗 +俩 +桩 +鑫 +钰 +榴 +皓 +罚 +國 +儒 +統 +荟 +搏 +笨 +烙 +腕 +锃 +袍 +筝 +肺 +悼 +陪 +酬 +憧 +憬 +盲 +罢 +涨 +艳 +俪 +渠 +滴 +蝼 +蛄 +卦 +趟 +咬 +鳞 +禄 +箫 +蝎 +丘 +厦 +侄 +畴 +牢 +蔽 +挖 +掘 +倡 +摹 +凸 +崖 +柏 +弥 +缆 +肃 +撮 +翌 +讹 +妨 +冤 +羔 +郎 +狭 +颂 +凱 +亞 +瑟 +扫 +茅 +纬 +夺 +抒 +犹 +蜀 +灭 +弓 +狮 +胞 +札 +啸 +瑚 +违 +伏 +蛙 +鸣 +吠 +澤 +镖 +舟 +烨 +晟 +滋 +翘 +绢 +牡 +璞 +腌 +瞬 +搁 +岱 +旨 +氯 +辄 +刨 +朕 +鲨 +媛 +琳 +凳 +韧 +匠 +逸 +愧 +聊 +昭 +颁 +勋 +篆 +肪 +凌 +扮 +宦 +逝 +拽 +哼 +凰 +灶 +洪 +揣 +匹 +叮 +锻 +墓 +鹤 +渍 +蛀 +颐 +琉 +耻 +辱 +暇 +籁 +掀 +怜 +惨 +呀 +戛 +翩 +膳 +娩 +愈 +坯 +逐 +醇 +坛 +喽 +赫 +蛤 +惫 +鞑 +虏 +橄 +榄 +斋 +舜 +昂 +扶 +炜 +裴 +迁 +昙 +曜 +坎 +狠 +诿 +咚 +叔 +柠 +檬 +喱 +酥 +咦 +咳 +咸 +玲 +堡 +驗 +際 +賽 +哆 +雇 +泣 +哮 +喘 +奸 +唤 +惰 +啃 +啄 +喔 +矶 +嗜 +嘿 +夕 +蘑 +雁 +熨 +烫 +專 +賣 +規 +衛 +題 +離 +苛 +炬 +捉 +拨 +糙 +雏 +轧 +愤 +怒 +驚 +匣 +湮 +仄 +窝 +埋 +撤 +匿 +棍 +泾 +垓 +垒 +宪 +髋 +墟 +殷 +拂 +壬 +癸 +馏 +蒜 +甫 +摁 +抚 +皙 +阙 +寡 +殆 +绑 +阱 +慵 +︰ +拙 +坟 +舀 +钗 +爷 +蚕 +盂 +吨 +採 +購 +優 +妞 +碣 +蜡 +阁 +佐 +镰 +乖 +谥 +吓 +桐 +豫 +凑 +扔 +坞 +雯 +悚 +妲 +侯 +诣 +婆 +娑 +滕 +裳 +孟 +羡 +蹴 +顽 +痒 +锥 +酚 +圭 +轼 +汴 +瞄 +绶 +宾 +耶 +猛 +邯 +扁 +脯 +豁 +堤 +樱 +歇 +援 +曳 +實 +繪 +編 +诙 +淆 +熙 +霖 +煮 +渗 +惕 +測 +間 +癒 +妩 +媚 +琥 +憨 +暄 +叛 +踵 +牲 +厉 +嘶 +膛 +毋 +瀚 +蔷 +薇 +撬 +屯 +崔 +嵇 +邢 +攒 +髦 +潭 +祁 +履 +阎 +迦 +牟 +挪 +萤 +嫉 +妒 +烁 +塘 +貔 +貅 +狄 +郊 +驹 +溥 +侗 +霆 +簸 +崩 +簿 +挣 +虔 +浑 +蛮 +鞭 +惹 +埂 +妹 +庵 +裔 +恕 +恳 +悍 +奢 +姨 +恼 +隆 +兆 +贰 +樵 +酝 +酿 +哒 +枉 +躯 +晴 +嘘 +憾 +勉 +窜 +殉 +浊 +踮 +蹑 +攀 +艇 +戳 +兩 +據 +資 +悸 +罄 +痧 +伶 +仃 +桌 +挫 +颦 +惮 +翱 +翔 +抛 +膝 +谭 +拷 +疚 +镐 +赃 +恪 +討 +搪 +郾 +跤 +擁 +钥 +匙 +谊 +隅 +敕 +诛 +诩 +學 +種 +選 +處 +挝 +迭 +羚 +戚 +跆 +囚 +奈 +爪 +扑 +潺 +斧 +毂 +瓢 +漳 +茬 +摘 +逵 +赴 +叵 +募 +俏 +寺 +曦 +洒 +阜 +霞 +叁 +睐 +趁 +卑 +坷 +醐 +弛 +窃 +辫 +牺 +盼 +決 +戰 +驶 +質 +渡 +垢 +镕 +槿 +钝 +蚂 +蚁 +熏 +钊 +氟 +幂 +澄 +隋 +炀 +⑤ +倦 +晦 +曙 +靖 +璇 +槃 +渎 +豚 +拯 +芭 +夾 +頭 +車 +細 +軸 +桔 +冶 +梵 +碍 +棵 +樊 +歸 +園 +書 +盎 +乒 +乓 +⊙ +視 +∕ +稽 +歪 +異 +終 +湛 +榭 +個 +誤 +屬 +魁 +痂 +毡 +鳝 +氦 +氣 +溴 +氮 +沾 +昵 +媲 +娟 +淮 +辙 +皖 +滩 +佤 +冕 +崛 +沦 +挞 +骏 +淳 +泗 +磬 +曰 +榆 +嵩 +炭 +澜 +蜃 +涓 +淝 +淞 +涮 +淺 +渤 +薰 +滥 +觞 +贻 +侨 +發 +積 +烽 +蕨 +棠 +庇 +牵 +萦 +犀 +猕 +馄 +饨 +悄 +嘻 +猶 +徘 +徊 +遠 +過 +傳 +奘 +怔 +诘 +綦 +跋 +汲 +尉 +浇 +漾 +腑 +嗽 +倚 +瑾 +俘 +Ｖ +侠 +咩 +苔 +辰 +孚 +觀 +亂 +厢 +槛 +盅 +Ｔ +宴 +闵 +荤 +脍 +炙 +當 +這 +樣 +贱 +瘟 +桦 +蛾 +灼 +璐 +褶 +伐 +﹐ +胥 +裕 +翳 +疔 +錄 +稣 +仇 +瞿 +畔 +鸠 +粪 +娌 +狩 +売 +菠 +屐 +藜 +厥 +贼 +芥 +耘 +龛 +钦 +爾 +動 +斩 +棘 +镳 +荨 +嫌 +舅 +欺 +妾 +谏 +痴 +凶 +恃 +役 +刁 +狡 +赐 +绫 +寝 +∶ +乾 +冥 +腦 +蹲 +璀 +璨 +冀 +罹 +殃 +靈 +詳 +粱 +瘫 +赎 +袈 +裟 +辖 +弈 +逞 +銮 +诏 +熬 +硒 +捐 +邂 +逅 +貂 +骷 +髅 +蝉 +谍 +寥 +娅 +給 +話 +詹 +砺 +趴 +汾 +妳 +绛 +矣 +仰 +秽 +侬 +狙 +翟 +捡 +喋 +仑 +咏 +麾 +琐 +赣 +蝇 +蟹 +胚 +婀 +绺 +腋 +拗 +粹 +胫 +哄 +芊 +皱 +樂 +笋 +檐 +殡 +葬 +苁 +戟 +苑 +苟 +戊 +岚 +溜 +萱 +莠 +莴 +苣 +荸 +荠 +陀 +鎏 +蜘 +蛛 +袂 +裆 +須 +覺 +遏 +瘋 +藝 +團 +睁 +蜊 +囧 +螃 +螨 +刍 +衍 +槟 +蚤 +芹 +闯 +誰 +萬 +歲 +诠 +诡 +谲 +吵 +脫 +庚 +嘟 +婺 +焯 +琛 +躬 +買 +劵 +旱 +吡 +啶 +臻 +婪 +绷 +卿 +澈 +胤 +锰 +沧 +級 +廚 +蛟 +躺 +嗝 +蓟 +饺 +贞 +寅 +賺 +哑 +匈 +弋 +轰 +衙 +罕 +豺 +惶 +侃 +嘈 +茗 +觑 +逛 +跪 +擒 +夷 +鱿 +邓 +讶 +邵 +彧 +懈 +楷 +僵 +溅 +馍 +钽 +锣 +羹 +酮 +翎 +製 +麥 +語 +匆 +躁 +盯 +俑 +呐 +隽 +擘 +睹 +歧 +竺 +瞟 +狰 +狞 +釜 +丐 +骐 +飲 +飽 +筆 +記 +飙 +嘎 +紅 +瑪 +咀 +铲 +杖 +馅 +鳄 +琪 +芪 +雌 +筷 +琶 +肇 +眶 +螳 +螂 +娓 +睬 +凛 +冽 +涌 +幢 +掐 +巿 +憋 +铿 +锵 +妄 +捲 +窠 +臼 +祗 +匪 +矗 +烩 +℉ +婧 +碾 +冉 +沼 +蕉 +燒 +抜 +込 +删 +薪 +抠 +铛 +Ⅰ +伝 +聞 +穗 +蜥 +蜴 +襄 +邑 +窦 +壹 +潞 +舂 +課 +巳 +咒 +醛 +溉 +潍 +笃 +螟 +赘 +骼 +鳗 +脓 +鸽 +羧 +饶 +峪 +竽 +耽 +肘 +腈 +槐 +礎 +應 +孢 +聂 +卯 +榻 +菁 +沅 +穩 +氙 +禹 +缪 +孜 +櫃 +鹦 +鹉 +杵 +調 +順 +宮 +備 +羟 +茯 +Ｌ +Ｃ +Ｄ +屎 +Ⅱ +^ +樹 +髮 +茧 +簡 +娶 +惟 +鲸 +荆 +渲 +暑 +戮 +嗨 +祇 +塬 +髻 +〖 +関 +獲 +畫 +兮 +冯 +張 +圖 +擾 +弼 +歡 +無 +爭 +寧 +靜 +辊 +颔 +歐 +暹 +芋 +馥 +缕 +砗 +磲 +竭 +戾 +褚 +吏 +硫 +吝 +啬 +惩 +晗 +嬉 +帷 +琵 +蟑 +營 +號 +眾 +憩 +疽 +骇 +巷 +蘭 +誠 +報 +惘 +論 +蓓 +巫 +靡 +缦 +卻 +難 +堰 +骅 +荚 +鸢 +崭 +姥 +淬 +舶 +呜 +怯 +腼 +腆 +構 +祕 +钯 +莹 +溯 +淤 +钣 +倜 +傥 +骚 +衩 +濒 +嚼 +逮 +腥 +迂 +尸 +箐 +荫 +並 +揮 +盡 +從 +鄂 +苞 +拴 +扼 +喉 +礁 +闸 +眺 +绰 +黝 +邃 +琅 +氓 +娆 +茸 +蚧 +樟 +贏 +業 +廠 +冈 +盹 +枢 +洱 +愣 +亢 +炽 +睦 +瞒 +咆 +躲 +關 +涕 +迸 +噬 +呛 +瘪 +祀 +飨 +缮 +茵 +堕 +祠 +卞 +潇 +筠 +巍 +摒 +粕 +隶 +匮 +桨 +嚏 +葆 +眷 +驳 +丶 +荔 +僚 +萼 +箭 +酌 +榷 +雎 +缀 +焉 +霹 +雳 +敝 +捂 +剁 +倘 +費 +電 +璋 +韓 +蝽 +眩 +Ｇ +凋 +钨 +谎 +捺 +撷 +叩 +幌 +挨 +叨 +Ｏ +揩 +叽 +哩 +〈 +〉 +嘀 +嗒 +鳳 +莘 +昶 +闺 +哎 +` +囤 +囔 +竖 +汞 +瘀 +喧 +哗 +痔 +肛 +焰 +怖 +藩 +彤 +瘿 +倫 +婷 +珐 +纭 +糅 +戌 +叼 +唏 +玺 +骊 +奎 +侈 +缇 +潦 +憔 +悴 +龚 +踝 +稼 +廖 +載 +銘 +漢 +妊 +娠 +妓 +婶 +俾 +豬 +攸 +署 +秸 +渺 +楦 +菖 +纰 +赌 +铋 +蹦 +懊 +蔼 +瞪 +拇 +菡 +嘛 +桓 +铠 +疳 +榫 +岫 +跻 +奄 +腓 +峨 +妍 +痘 +斥 +屑 +偕 +彪 +苷 +吁 +昇 +铢 +驯 +徙 +庶 +見 +馬 +內 +闽 +紋 +蜕 +鹃 +驷 +慑 +钵 +咙 +镣 +铐 +琼 +茜 +幔 +洮 +寵 +薦 +軍 +敞 +領 +峥 +嵘 +俎 +斡 +秃 +廣 +噻 +藥 +態 +蟆 +磊 +謊 +恺 +惦 +诧 +霾 +啮 +谔 +晔 +袄 +辍 +叱 +咤 +郝 +佯 +噶 +Ｑ +鹫 +梓 +組 +賦 +親 +甾 +桖 +芷 +凄 +愫 +珏 +汝 +邺 +沁 +泫 +亩 +铒 +浣 +霓 +涿 +蔚 +嫔 +黔 +炊 +蒡 +筐 +馋 +捅 +玖 +柒 +哇 +灣 +鸳 +鸯 +砧 +痢 +沏 +涣 +馒 +骠 +畲 +確 +達 +戶 +柚 +啫 +谬 +踞 +廓 +較 +雖 +鬥 +颀 +塌 +睑 +颊 +睾 +夠 +傷 +潢 +結 +椿 +窈 +窕 +蹊 +漩 +鲈 +臧 +巽 +疱 +裘 +诃 +沌 +腴 +谶 +拱 +肱 +腱 +髂 +奚 +疟 +毽 +峭 +梢 +穹 +隍 +嗳 +渥 +柿 +缰 +嘌 +呤 +認 +敗 +毗 +酋 +捶 +艮 +珈 +芍 +芸 +茉 +薏 +蘸 +耸 +頁 +吭 +觊 +觎 +疙 +瘩 +蛊 +請 +針 +膚 +瑩 +觅 +摞 +寐 +遨 +嚣 +讥 +绯 +淌 +叟 +痤 +婢 +莺 +掷 +饵 +沮 +貴 +價 +猝 +岐 +識 +鳴 +Ｅ +発 +麂 +舷 +輕 +舵 +辜 +艘 +瞰 +沱 +涎 +遁 +兀 +禀 +嫂 +芮 +笠 +還 +蜻 +蜓 +邱 +焘 +镙 +幡 +猗 +歹 +朔 +昔 +恣 +谑 +韬 +燮 +舫 +霏 +痊 +扦 +箕 +砼 +痨 +骶 +钒 +黃 +筏 +岔 +帛 +唬 +亳 +Ｘ +膦 +劇 +塾 +苋 +甬 +扒 +吩 +咐 +碘 +苡 +吋 +熵 +驅 +陰 +磺 +锛 +枳 +浔 +麿 +誘 +連 +舊 +擋 +咔 +唑 +鼹 +肴 +桅 +秧 +損 +潔 +劑 +钼 +芩 +湯 +麵 +垄 +漱 +門 +網 +狀 +⑧ +█ +幫 +碩 +紊 +隱 +搐 +斟 +咎 +荅 +皋 +拈 +評 +爲 +臺 +農 +產 +標 +丞 +茴 +蔺 +蝗 +肆 +虐 +墩 +靳 +袤 +崂 +豌 +甄 +砾 +獭 +勐 +懿 +铵 +華 +慨 +宸 +壑 +磋 +讫 +僅 +釐 +鳕 +僻 +镑 +恬 +栀 +逍 +薹 +汕 +膠 +昱 +獐 +膺 +侏 +黯 +荞 +啼 +诰 +嫖 +陡 +俚 +亘 +篡 +莼 +砥 +汶 +佘 +夥 +宕 +谴 +牒 +洼 +霭 +阉 +镭 +俐 +貧 +阖 +傣 +豉 +吒 +鋼 +線 +條 +膩 +锢 +郵 +彷 +徨 +沖 +胛 +仨 +锗 +汹 +谖 +荃 +飓 +卅 +飒 +恍 +惚 +郡 +鞠 +攘 +邸 +黜 +桎 +梏 +牍 +減 +慾 +嗦 +弩 +铆 +蹿 +則 +禮 +哔 +崤 +铰 +﹑ +蚌 +惡 +旷 +匕 +堅 +強 +祉 +咫 +夯 +舱 +稔 +睽 +淄 +膑 +獾 +夭 +簌 +蝌 +蚪 +吮 +煨 +殇 +蜉 +蝣 +鋪 +逗 +佗 +掬 +慷 +徕 +赦 +熄 +愠 +嬷 +撸 +煦 +杲 +窒 +莽 +泯 +紛 +籠 +腩 +蝈 +斛 +驮 +阡 +遐 +∥ +苇 +熹 +胭 +嚎 +毓 +氰 +鞅 +梧 +槁 +呻 +雜 +誌 +蕙 +瞳 +芘 +痞 +龋 +琏 +焖 +牦 +鹩 +祟 +迥 +簪 +玥 +歼 +瓯 +蘖 +茁 +痉 +垩 +喳 +飛 +稷 +訊 +酉 +恆 +讀 +鸾 +赊 +弑 +壅 +绦 +祷 +魇 +哨 +粟 +嶙 +峋 +渝 +伉 +涧 +懦 +瘸 +蕻 +煜 +楫 +豐 +聯 +絡 +謝 +赅 +钿 +螈 +虛 +攝 +補 +蘇 +夔 +舔 +纂 +樽 +赝 +蕃 +霁 +蹇 +傀 +儡 +烊 +擞 +冢 +赡 +悅 +窘 +陇 +枋 +鲣 +笆 +汜 +篱 +雲 +闳 +骡 +鹑 +復 +龅 +繫 +呕 +懋 +區 +譽 +寫 +擢 +簇 +戎 +麸 +腧 +東 +揍 +暨 +唧 +傑 +晉 +暢 +銷 +Ｓ +椽 +況 +咄 +厩 +湍 +蛭 +楔 +碁 +擬 +彙 +螢 +蟲 +笈 +霄 +骰 +呦 +吳 +單 +顆 +鎖 +缉 +岖 +壕 +垣 +眯 +虢 +鹌 +遊 +戲 +Ｕ +吱 +椴 +泠 +錾 +財 +榛 +圃 +蜈 +蚣 +鹭 +蜷 +蜍 +笞 +弭 +缥 +缈 +婕 +佼 +揖 +拮 +茭 +臆 +恻 +镌 +紹 +瞥 +嚷 +倏 +皎 +俨 +維 +護 +蘅 +荪 +喃 +鲲 +炳 +晤 +剽 +犷 +啾 +鏖 +囿 +屌 +譬 +貉 +寮 +蛹 +唛 +腫 +熱 +伢 +掸 +輔 +導 +響 +殼 +娲 +撇 +寰 +酗 +雉 +瘙 +猬 +囡 +掏 +乍 +彗 +讴 +菏 +淹 +項 +産 +婦 +啰 +羁 +咿 +唷 +覽 +鹊 +孑 +聲 +捌 +摧 +納 +隨 +饷 +‖ +犸 +谧 +朦 +胧 +飾 +岀 +蜿 +蜒 +荏 +殒 +嗓 +訂 +詢 +瞅 +莅 +譯 +驟 +檢 +診 +挠 +紙 +屿 +涟 +漪 +別 +鳍 +铱 +迩 +蓑 +濮 +瘠 +咛 +煩 +惱 +宓 +氤 +氲 +荻 +唢 +笺 +煅 +鲢 +鳙 +噱 +龈 +禾 +贲 +躍 +犁 +膈 +谒 +狈 +镗 +奧 +骈 +衄 +悯 +姊 +骥 +貼 +緒 +蕲 +侮 +垠 +肋 +嵴 +緩 +壓 +渚 +阕 +瓤 +倭 +銀 +變 +橫 +瓠 +篝 +谤 +隘 +缭 +轉 +參 +镊 +係 +糠 +钕 +璟 +锲 +盤 +Ｆ +莨 +灏 +赓 +鹧 +鸪 + +糜 +廳 +呸 +裏 +絕 +坂 +暧 +脘 +峦 +崽 +帳 +稅 +啉 +叢 +闰 +Ⅶ +涝 +稗 +試 +験 +痈 +熠 +捩 +搂 +宵 +恙 +該 +織 +⑥ +鲇 +螯 +钡 +範 +邊 +畢 +鳌 +暮 +铎 +館 +捍 +劍 +癜 +殺 +仟 +诅 +監 +楽 +浚 +媳 +棣 +垦 +痣 +楣 +倔 +跷 +匾 +喰 +鬓 +週 +衝 +聖 +綜 +犊 +忱 +骸 +憎 +忒 +屹 +谙 +祐 +笙 +澍 +敖 +庖 +咋 +蒹 +葭 +坳 +蔑 +麗 +痪 +耙 +礙 +淨 +島 +韶 +捎 +邬 +蛔 +疥 +癞 +嫣 +翦 +啪 +嚓 +绗 +搀 +執 +龍 +兒 +踱 +岌 +撵 +俺 +锄 +诂 +擺 +芾 +帶 +慶 +筵 +棺 +鍵 +濠 +珩 +徜 +徉 +蛰 +惭 +習 +掛 +濑 +嘗 +漕 +阪 +磐 +懑 +糸 +馊 +窿 +癖 +蚜 +榔 +蓖 +阮 +丨 +锶 +蟠 +锭 +湄 +滁 +藓 +淼 +斫 +撩 +嬛 +髁 +癎 +贬 +鬆 +菸 +瓮 +痿 +謀 +鄙 +忏 +椹 +芜 +姗 +髯 +虬 +赭 +頫 +孺 +丕 +階 +擇 +郴 +摺 +蟒 +滢 +員 +狒 +绉 +纣 +篓 +﹝ +﹞ +譜 +悌 +Ｉ +Ｂ +Ｎ +Ｍ +袒 +攫 +茼 +偉 +唠 +劃 +臊 +箝 +瞌 +疣 +糧 +隻 +砷 +嗲 +證 +Ⅺ +螭 +顏 +惋 +溝 +兢 +粼 +昉 +陉 +泞 +紗 +預 +収 +冊 +齊 +膻 +辕 +痼 +琬 +猾 +峙 +Ｐ +礴 +畿 +纨 +勞 +厮 +禺 +轶 +淦 +幾 +皲 +谕 +楝 +蛆 +澎 +湃 +墉 +玮 +雹 +蚬 +胄 +锹 +陂 +剐 +鄞 +跡 +眈 +麓 +遛 +穰 +苴 +協 +扈 +缢 +蛏 +鲟 +蛎 +瞠 +帚 +臉 +蟋 +蟀 +郸 +驿 +氐 +襁 +褓 +驸 +赈 +锺 +镲 +孰 +韫 +蠕 +忻 +踹 +唆 +恸 +汆 +腮 +烏 +瀛 +訓 +練 +飕 +栢 +榈 +陨 +殚 +養 +濕 +欖 +馀 +茹 +鈴 +溟 +饴 +吲 +哚 +賞 +藍 +顷 +茱 +蓮 +枷 +佃 +盥 +燙 +雙 +換 +诋 +鬃 +犒 +铨 +缨 +嫡 +擂 +楊 +蘿 +蔔 +遴 +飚 +邋 +遢 +跖 +聩 +袅 +喙 +闫 +嗮 +铳 +冼 +純 +炯 +負 +責 +憶 +礦 +餃 +˙ +鳅 +氽 +旎 +谟 +議 +顯 +權 +嬴 +纾 +玑 +複 +聰 +闾 +熘 +奂 +哧 +皈 +璜 +倬 +錢 +悖 +翊 +苻 +饯 +钏 +廟 +輸 +祜 +鲂 +镉 +亟 +犟 +貓 +邰 +娣 +側 +浜 +簽 +Ⅹ +喆 +註 +釋 +噼 +诽 +袱 +胯 +绥 +臃 +伎 +匝 +攥 +钜 +樾 +興 +遞 +愕 +鎮 +搽 +徳 +綱 +榉 +踌 +躇 +桉 +苄 +膽 +姣 +吆 +沓 +铌 +獒 +畦 +縫 +牠 +滾 +阆 +渭 +Ⅴ +廪 +嚴 +侶 +窮 +帼 +羅 +聽 +懵 +臨 +蝰 +蜱 +蠡 +褒 +嶂 +莊 +汐 +裨 +钎 +晷 +燧 +Ⅸ +菀 +鳜 +荀 +珅 +蚓 +繡 +肮 +査 +檔 +講 +崴 +実 +坍 +Ⅷ +萸 +陳 +▽ +閱 +欄 +睢 +撂 +邕 +颞 +醬 +酊 +侥 +甥 +蠹 +骛 +蹒 +跚 +鞣 +綁 +呗 +瘘 +痱 +孛 +貝 +跛 +韻 +岑 +麝 +腸 +桿 +鳥 +捨 +棄 +擴 +矜 +揚 +孪 +姒 +轱 +辘 +枇 +杷 +柘 +绡 +轫 +Ｋ +Ａ +Ｒ +歷 +輝 +獻 +橋 +鹳 +捭 +￠ +淚 +澂 +晁 +苜 +蓿 +闩 +桁 +耷 +︱ +烬 +販 +颛 +鹂 +漉 +晌 +鬣 +醪 +馕 +禍 +帙 +願 +鹘 +煽 +Ｙ +Ｚ +颧 +怦 +髌 +榧 +檫 +妝 +勝 +铑 +⑨ +鮮 +燉 +啜 +啕 +瓒 +绮 +芫 +荽 +檸 +钺 +汛 +﹒ +皑 +栉 +嬗 +苫 +閒 +潛 +珞 +娄 +炕 +坨 +潼 +缜 +劉 +〞 +锱 +馁 +鹗 +刈 +凫 +髎 +颢 +謇 +逶 +迤 +舉 +酐 +軟 +涪 +偈 +臟 +蹩 +猷 +篦 +锴 +娉 +衅 +賓 +蚯 +蝮 +擔 +職 +媽 +铮 +鐘 +齡 +猄 +谡 +黍 +‐ +萘 +煸 +鍋 +矽 +鉻 +⑦ +桧 +陛 +俳 +荥 +幄 +呱 +勢 +葉 +邳 +聃 +繇 +綿 +徹 +罡 +跺 +嵊 +霎 +閉 +裾 +諸 +隊 +≧ +≦ +搖 +哕 +緣 +讷 +龌 +龊 +藿 +匍 +匐 +埚 +諾 +瞭 +涸 +芈 +姝 +缙 +颍 +莜 +衢 +诬 +葚 +濃 +廢 +肫 +慣 +鲛 + +芎 +萜 +脲 +掳 +鐵 +殴 +蓼 +潴 +箩 +屆 +獎 +剛 +頻 +掖 +旖 +輩 +煊 +擀 +妤 +蜣 +濟 +漸 +哌 +顛 +榮 +挈 +魯 +蹂 +躏 +诤 +饋 +掣 +嘹 +咧 +涼 +鄉 +夢 +燎 +怆 +拚 +椁 +嗔 +輪 +涔 +锷 +踐 +陣 +禳 +鄱 +衾 +鍛 +鍊 +廿 +栎 +诟 +臌 +疖 +藐 +粽 +賀 +芡 +觸 +柞 +褂 +哞 +餅 +絲 +恽 +俸 +層 +揪 +茲 +潑 +阂 +貢 +詩 +垅 +碉 +壞 +骞 +骁 +珉 +単 +樨 +鞘 +罔 +呃 +囫 +陲 +蒯 +擊 +鏈 +婿 +対 +攪 +傾 +琨 +嗎 +戍 +橇 +馗 +赳 +崧 +诌 +偎 +麋 +舸 +逋 +嗖 +埭 +贈 +嗪 +徑 +偌 +腳 +虻 +佞 +谗 +佝 +偻 +審 +皴 +潸 +杈 +鄢 +碴 +檄 +婵 +唁 +瘁 +杳 +缄 +闆 +迴 +楹 +镛 +誕 +魃 +頂 +澧 +炝 +遷 +濡 +爻 +旌 +捋 +舐 +墊 +蓦 +颓 +縱 +劬 +薮 +郗 +≮ +≯ +訴 +築 +荼 +伫 +歆 +邈 +緊 +嘚 +詮 +嗤 +吖 +甙 +蛲 +獗 +蛐 +駕 +仞 +覃 +锕 +円 +閃 +樓 +罂 +様 +醴 +谆 +圄 +旸 +蚩 +埙 +徭 +喬 +塊 +煙 +椐 +諧 +喹 +聿 +酞 +硐 +粲 +骧 +瓴 +雞 +錯 +幣 +牆 +飯 +唰 +億 +抨 +噌 +險 +羿 +遒 +塗 +幹 +抡 +耆 +搔 +黒 +嗷 +薩 +娼 +醺 +勻 +讚 +宥 +彿 +谄 +髙 +Ｊ +楮 +胗 +桀 +褛 +颚 +壽 +蔥 +啧 +п +я +咂 +轲 +迢 +琮 +額 +舛 +颏 +溏 +囟 +纖 +鏡 +獠 +抻 +谩 +鼬 +淖 +鲆 +啖 +炷 +槌 +劾 +蓋 +樑 +貫 +訪 +競 +鹹 +攜 +沢 +瘰 +疬 +笫 +觐 +説 +郦 +苎 +穀 +遺 +駐 +啟 +鴉 +壯 +槭 +琰 +霈 +蓍 +臬 +儀 +趨 +暫 +蘋 +蕹 +祂 +簋 +黠 +葳 +邁 +咝 +惺 +忡 +菅 +蒌 +薤 +鍾 +嘭 +怏 +羯 +瞽 +醍 +靛 +祚 +窖 +焱 +綠 +糾 +莳 +顼 +仝 +岷 +猖 +鲷 +諮 +顔 +蘊 +憑 +醌 +嗡 +鲶 +穑 +塵 +撫 +谪 +陸 +钤 +耄 +耋 +鲵 +幺 +苒 +蚵 +餡 +涞 +◣ +◢ +鑑 +陟 +楸 +踉 +跄 +帏 +Ｗ +拄 +蝦 +騎 +栝 +箧 +挲 +燊 +绾 +剋 +阗 +瘕 +祿 +夙 +蒐 +泱 +Ｈ +饬 +獅 +鲽 +猞 +猁 +呂 +蕅 +椤 +懼 +庾 +闱 +沣 +辭 +違 +齒 +鹽 +紐 +峒 +琊 +賴 +卟 +癔 +弁 +掇 +谌 +偃 +醣 +鲳 +褴 +杼 +災 +専 +砜 +遜 +辺 +唎 +呓 +掂 +倉 +縣 +梆 +勁 +锑 +気 +嗟 +証 +唔 +銅 +裥 +蔫 +濂 +呋 +◤ +傢 +囍 +萊 +爛 +曬 +绻 +胝 +> +· +昀 +° +& +← +↓ +縻 +" +” +騄 +“ +≤ +≥ +< +馝 +² +褯 +√ +淜 +凍 +幚 +綫 +觇 +醮 +艏 +鲗 +脟 +³ +蒱 +罽 +鈣 +豊 +愎 +柅 +妯 +膘 +¥ +钍 +尕 +虓 +凈 +‘ +’ +噔 +頌 +± +醨 +毵 +崋 +絨 +玃 +葖 +… +— +菪 +䏡 +陬 +檞 +洇 +汦 +缬 +扱 +瀣 +祋 +Ω +尻 +骉 +μ +→ +殁 +腽 +燚 +衮 +粧 +鷉 +薬 +≈ +冁 +騵 +謏 +乜 +傘 +勖 +勲 +嚭 +砹 +夐 +郜 +黼 +䗛 +嵗 +鄀 +牖 +㭎 +▪ +豳 +▶ +樋 +鳃 +◀ +黇 +輋 +麽 +葎 +砵 +穄 +甭 +綄 +洌 +芤 +硚 +酹 +籤 +﹣ +× +俍 +鏏 +↑ +鵏 +眞 +嬂 +≠ +盜 +緯 +縝 +溆 +亓 +騠 +砬 +鐍 +‥ +栊 +翮 +鲘 +藟 +墖 +棐 +粝 +迺 +趼 +吣 +庠 +鮭 +纇 +満 +骺 +睨 +黉 +脞 +媄 +胨 +郓 +濋 +圜 +繄 +锫 +滆 +澫 +潏 +罘 +譓 +叻 +殳 +癢 +哐 +皛 +偼 +诮 +鼽 +䕶 +窨 +䡵 +疁 +遘 +钚 +卺 +肟 +擿 +鹼 +鄗 +彳 +醯 +雱 +貯 +鮑 +坻 +缁 +瓀 +噁 +髪 +鳯 +煳 +嵫 +伈 +窀 +堃 +軒 +豕 +礌 +晳 +• +锏 +骦 +晡 +嬿 +朸 +棪 +铧 +勰 +缧 +囵 +駓 +迕 +↘ +↖ +岡 +镩 +齉 +氡 +尨 +逑 +鰤 +珮 +怃 +譲 +婤 +譩 +譆 +膂 +滧 +踺 +∞ +耢 +豨 +勠 +孅 +枅 +殄 +馭 +噠 +鄑 +賜 +嬰 +鈍 +鈿 +臍 +噙 +朐 +苤 +垈 +嬄 +饧 +雠 +菰 +辂 +嵽 +甍 +榅 +閏 +繃 +蘆 +薈 +汙 +腒 +冮 +佽 +黾 +隺 +脬 +铏 +€ +梠 +鏽 +噴 +霧 +崶 +脅 +餾 +絜 +佸 +勵 +鮈 +蔄 +畤 +缐 +洓 +躅 +绀 +㛹 +诶 +犄 +汋 +忖 +慝 +⑩ +枼 +垕 +搌 +翂 +铷 +阄 +菓 +廛 +肓 +龠 +滪 +劓 +玟 +缛 +饳 +嵛 +∧ +蟎 +虒 +絷 +龆 +鲰 +翾 +秕 +纕 +镬 +戕 +寳 +抿 +舖 +詈 +繩 +牽 +艹 +纆 +儉 +愍 +躞 +娵 +镪 +羕 +龉 +镝 +罴 +硌 +輗 +菂 +犇 +暲 +鑒 +輦 +镵 +締 +儲 +䲢 +诹 +軀 +喚 +ˉ +扂 +篢 +贜 +ω +姘 +烔 +爰 +圉 +↙ +↗ +彄 +跬 +傒 +釿 +袢 +跗 +諟 +绐 +骜 +▏ +腚 +旐 +廋 +橦 +隹 +囹 +雊 +邙 +绖 +戣 +刖 +筻 +￡ +綎 +赗 +氍 +舄 +檑 +鼗 +償 +遲 +閣 +囲 +鲊 +潵 +甌 +慥 +瘛 +荇 +鐽 +▍ +∣ +悃 +蹐 +鹾 +鹜 +娈 +蹉 +扺 +欹 +貘 +碓 +锖 +薁 +鼱 +锍 +杓 +兿 +惝 +厖 +劼 +橛 +稂 +酅 +猃 +芼 +糒 +硖 +磡 +憕 +弨 +脔 +龃 +揕 +嵖 +﹢ +旆 +镨 +骝 +褊 +郚 +歅 +笪 +纛 +牥 +薳 +镞 +荬 +锧 +诜 +瀼 +孳 +斠 +圊 +骓 +︿ +袪 +雒 +趯 +珸 +捃 +﹥ +秬 +嫒 +蹁 +跹 +鹛 +吥 +鼍 +舯 +卬 +淙 +羼 +酽 +嘞 +嘧 +苾 +諓 +眄 +騊 +颙 +鄚 +蘘 +潋 +滟 +鳊 +觭 +禋 +闉 +盍 +哳 +蚰 +骟 +裣 +扃 +宄 +嫚 +袼 +蹙 +栲 +∈ +鐡 +饅 +嗄 +郿 +炣 +塰 +蛳 +熥 +瑄 +枞 +俢 +芨 +妏 +埕 +晞 +嬅 +媞 +萩 +倮 +価 +瀾 +琯 +珺 +暘 +偮 +玎 +襪 +璎 +祎 +驲 +蔪 +α +晢 +嬤 +屛 +姹 +銥 +璽 +妡 +転 +咲 +柰 +苘 +牪 +洧 +绋 +澋 +鷄 +盱 +眙 +袆 +鈷 +䂀 +苕 +旻 +珰 +轳 +訫 +咭 +莵 +噗 +浠 +ˆ +婊 +湓 +嗑 +孬 +倆 +袴 +瀘 +饽 +▫ +芃 +瀞 +熳 +蒼 +煖 +杺 +咻 +㧴 +壇 +嵮 +桤 +兲 +灆 +塢 +铪 +瘊 +孖 +萆 +薢 +蔾 +凔 +棧 +葐 +﹩ +哓 +哂 +柫 +缟 +垚 +埗 +洹 +厝 +玙 +衿 +佲 +槎 +岙 +峽 +綺 +謎 +蒄 +曈 +塍 +荳 +佧 +砀 +刄 +駿 +蓢 +圩 +捘 +阊 +悻 +狍 +郞 +偲 +伲 +淏 +唻 +臜 +塱 +郫 +浉 +睏 +疃 +笕 +畈 +弇 +縁 +茌 +憇 +紡 +昄 +啵 +浐 +肈 +谞 +俹 +沩 +柁 +俫 +镆 +垟 +洺 +箬 +濛 +鏌 +茡 +苪 +瑷 +団 +茆 +槙 +蕗 +糗 +邝 +褟 +铉 +鲱 +俣 +滐 +咮 +蠟 +鲅 +沭 +鲋 +漯 +蒽 +灥 +镫 +峄 +湑 +俁 +宬 +酩 +珲 +铖 +塚 +戦 +郯 +瑀 +怂 +禤 +錡 +蔲 +锜 +煕 +碶 +蘼 +赑 +繆 +僖 +莒 +㚢 +菘 +鸲 +渃 +β +扪 +谵 +橼 +螫 +蕈 +猥 +亵 +疋 +蚴 +眦 +襞 +胼 +绌 +擤 +腠 +纡 +胬 +蹼 +霰 +襻 +榼 +蒺 +岢 +脒 +莪 +葶 +莶 +巯 +胍 +菴 +醑 +癀 +檗 +肼 +癃 +癥 +菥 +蓂 +蝥 +酏 +鳢 +泮 +鉍 +荜 +茇 +菝 +葜 +薷 +蟅 +尪 +腙 +芴 +蒎 +棓 +楤 +呫 +Ι +γ +胂 +蠲 +茚 +镧 +礞 +艽 +艿 +苈 +蓣 +钆 +鳔 +鸬 +鹚 +鸶 +φ +δ +缗 +铯 +羸 +鉧 +毳 +啘 +喑 +′ +噤 +噫 +耵 +聍 +矇 +搦 +忤 +瘅 +瘵 +疰 +痃 +瘴 +蛉 +劄 +睥 +瞀 +羰 +聤 +廔 +腘 +臁 +蓐 +铊 +蟥 +黧 +銹 +澆 +鋁 +錫 +閑 +絢 +搶 +鄭 +擲 +遙 +鈔 +鑽 +濾 +錦 +貿 +輻 +盞 +渦 +攤 +灘 +績 +嘆 +訟 +脹 +帥 +掃 +鈎 +闊 +褲 +襯 +屜 +櫥 +慮 +賠 +擠 +騰 +瀝 +賢 +瓊 +悶 +脈 +箏 +皺 +奮 +槍 +爐 +嬌 +韌 +瞞 +駛 +膿 +瘡 +滌 +氫 +憂 +釀 +誼 +疊 +鴨 +蠶 +鵝 +罵 +睞 +噸 +鞏 +嗆 +緬 +攏 +謙 +縛 +繞 +蕩 +鑰 +罰 +叄 +貳 +崗 +軌 +銳 +趙 +滬 +剝 +鄧 +諒 +濺 +繽 +紉 +賃 +鑲 +頑 +氈 +艱 +釣 +絞 +綉 +瀏 +頸 +撲 +殘 +腎 +賬 +厭 +鶴 +獸 +暈 +橢 +寢 +翹 +謹 +鴻 +艦 +墜 +穎 +懶 +敵 +朧 +鈉 +馳 +謠 +嶺 +饒 +喲 +啞 +瀉 +瑣 +蔣 +窩 +紳 +墳 +蝸 +螞 +蟻 +豎 +鈕 +撥 +賈 +濁 +襲 +俠 +煥 +鉛 +艷 +黨 +錐 +彎 +煉 +憐 +毆 +鬧 +諱 +毀 +滅 +鈞 +獄 +匯 +陝 +馮 +闖 +趕 +龜 +顫 +濱 +曉 +蠅 +漿 +輿 +簾 +喪 +鍬 +燦 +債 +憲 +棗 +鋅 +辯 +鋒 +虧 +釘 +窯 +飄 +廈 +熒 +錘 +磚 +閥 +槳 +贊 +鑄 +懸 +臘 +遼 +闡 +樺 +綴 +鍍 +滯 +貸 +矯 +肅 +漲 +壺 +鯽 +漁 +厲 +偵 +邏 +蝕 +攔 +摳 +禱 +誨 +飼 +淵 +騙 +滲 +閨 +凜 +鉀 +潰 +綽 +齋 +聳 +轎 +蕭 +癬 +纜 +狹 +錳 +鷹 +彌 +爺 +鯊 +廂 +籃 +偽 +頒 +綳 +誦 +禪 +鐺 +娛 +滷 +鱗 +龐 +墾 +蠻 +灑 +墮 +窺 +鄰 +嬸 +囑 +籌 +轟 +緞 +侖 +嶼 +頗 +鉗 +驕 +屢 +騷 +憤 +摻 +豈 +硯 +絹 +燭 +瀟 +盧 +爍 +摯 +憫 +斂 +顱 +縷 +轄 +贅 +貞 +雛 +嚇 +賊 +蕪 +濤 +輛 +殲 +犧 +疇 +嚨 +攆 +隸 +貪 +詠 +僑 +拋 +篩 +鏟 +驢 +檻 +綻 +滄 +癮 +諷 +辮 +獰 +葦 +饑 +餓 +澀 +竊 +頰 +餌 +尷 +奪 +繭 +鴿 +淪 +粵 +掙 +覓 +簫 +閩 +駱 +駝 +鵬 +憊 +瘧 +擰 +嶄 +楓 +勸 +詭 +棟 +賤 +謬 +睜 +繹 +銜 +嘯 +閘 +濫 +攬 +鴕 +綢 +撿 +樞 +艙 +壟 +隕 +蔭 +諜 +饞 +欽 +擱 +嘔 +譚 +慘 +鯨 +懇 +懲 +誇 +鋸 +蟬 +罷 +鰐 +棲 +斃 +鐮 +莖 +巔 +馱 +撈 +狽 +鸚 +鵡 +糞 +幟 +渾 +摟 +畝 +詐 +鵑 +諺 +壘 +樁 +葷 +譏 +澗 +纏 +誡 +嗚 +駁 +駭 +貽 +鴛 +鴦 +囂 +鯉 +藹 +輾 +繳 +襖 +賭 +嘩 +錨 +婁 +譴 +鵲 +曠 +駒 +籮 +竄 +靂 +贛 +撓 +斬 +廬 +僥 +堯 +饃 +鑼 +頃 +搗 +壩 +鶯 +勛 +鱉 +軋 +鑿 +轍 +釁 +賄 +簍 +騾 +聶 +訝 +矚 +頹 +貶 +閻 +鷗 +繮 +馴 +鎊 +蹺 +慚 +礫 +嘰 +絆 +鬢 +癟 +礬 +聾 +鰭 +繚 +鋤 +猙 +濘 +巒 +癱 +晝 +嘮 +籬 +謗 +緝 +穢 +虜 +諄 +稟 +瘓 +瀕 +渙 +洶 +贖 +揀 +嶇 +鎬 +骯 +醞 +攙 +澇 +餒 +銬 +贓 +挾 +訛 +賂 +誣 +窪 +邗 +滦 +埇 +郧 +浈 +堽 +垸 +珙 +邛 +崃 +岘 +垭 +濉 +牯 +稜 +碚 +谯 +浯 +兖 +凇 +芗 +耒 +灞 +渑 +渌 +甪 +邡 +滘 +浍 +潆 +傈 +僳 +磴 +仵 +泺 +紶 +葑 +藁 +迳 +垌 +泷 +矸 +墒 +篁 +汊 +幛 +鄄 +漖 +秭 +儋 +湟 +枧 +厍 +崮 +砦 +犍 +鄯 +赉 +郏 +秣 +隰 +尓 +垱 +蓥 +呷 +蕰 +坭 +汨 +瀍 +沔 +垻 +畹 +蚶 +鮀 +辇 +沚 +垴 +垡 +淅 +埯 +剡 +桷 +埒 +邨 +圪 +夼 +仡 +猇 +堍 +洸 +溱 +胪 +湫 +洄 +笏 +辋 +郢 +亍 +崆 +泖 +珥 +黟 +濯 +昝 +滏 +洎 +埝 +窢 +褔 +沤 +峤 +沨 +乂 +餍 +铦 +嵬 +赟 +訾 +墁 +锟 +逄 +肸 +姵 +劭 +璁 +酆 +贠 +玢 +邴 +弢 +臾 +薽 +牮 +戢 +靑 +曌 +隗 +翀 +洨 +鋆 +倧 +鍏 +璠 +贇 +钫 +赇 +珵 +錟 +炅 +頋 +珣 +粦 +尡 +璩 +婳 +甑 +篤 +镔 +嶷 +洐 +焜 +樯 +舲 +媗 +竑 +嫄 +澣 +甦 +麴 +娚 +琎 +韡 +钇 +謤 +骢 +穸 +眭 +唅 +鄷 +畇 +啹 +峯 +汔 +妎 +婂 +萏 +秝 +芄 +叡 +赜 +湉 +缑 +敻 +娒 +嘨 +礽 +缊 + +傛 +阚 +澹 +昳 +琚 +翥 +翚 +昦 +逯 +飏 +琤 +勍 +钭 +椋 +湧 +囝 +翃 +镱 +堉 +庹 +瑗 +钲 +怿 +劢 +镄 +岍 +琍 +崟 +鹈 +ā +á +ǎ +à +ē +é +ě +è +ī +í +ǐ +ì +ō +ó +ǒ +ò +ū +ú +ǔ +ù +‰ + +¢ +£ +฿ + + diff --git a/checkpoints/net_prior_860000.pth b/checkpoints/net_prior_860000.pth new file mode 100644 index 0000000000000000000000000000000000000000..4eb67e5d607e079bca5a912fe1f7dc1350f9da7f --- /dev/null +++ b/checkpoints/net_prior_860000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca229443cdbbd3dbe0fbaed2ffb77d5781473fa65bd8286408546a389e44e48c +size 111897207 diff --git a/checkpoints/net_sr_860000.pth b/checkpoints/net_sr_860000.pth new file mode 100644 index 0000000000000000000000000000000000000000..f1f0dfcc01a7fc809e325ea709f792dd496c8e9f --- /dev/null +++ b/checkpoints/net_sr_860000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f06c9d7e195c529015fee9ef5f14f7b501ebeaba42f8baecdd86694bafd00841 +size 67812621 diff --git a/checkpoints/net_w_encoder_860000.pth b/checkpoints/net_w_encoder_860000.pth new file mode 100644 index 0000000000000000000000000000000000000000..e95c09dfd06f77cdd17da8748d9191333692fd8e --- /dev/null +++ b/checkpoints/net_w_encoder_860000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c2626a4d3998b3edd88c60c2cdada721ea7531b366484c20a874819cbb426e5 +size 87585199 diff --git a/checkpoints/yolo11m_short_character.pt b/checkpoints/yolo11m_short_character.pt new file mode 100644 index 0000000000000000000000000000000000000000..0304cedcb3cb61ba1eea00c66a62b80a69c89b1e --- /dev/null +++ b/checkpoints/yolo11m_short_character.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af7e0d4a549ad399065fc2f21d06688fcaa24b3d0d16940ebbcf37d041b14c55 +size 80746880 diff --git a/models/TextEnhancement.py b/models/TextEnhancement.py new file mode 100644 index 0000000000000000000000000000000000000000..d6877be311c9bc6a11f92e2a997d8d96ea8a72f0 --- /dev/null +++ b/models/TextEnhancement.py @@ -0,0 +1,367 @@ +# -*- coding: utf-8 -*- +import cv2 +import os.path as osp +import torch +import torchvision.transforms as transforms +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import logging +logging.getLogger('modelscope').disabled = True + +from cnstd import CnStd +from utils.utils_transocr import get_alphabet +from utils.yolo_ocr_xloc import get_yolo_ocr_xloc +from ultralytics import YOLO + +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks +from networks import * +import warnings +warnings.filterwarnings('ignore') + +from modelscope import snapshot_download + + +########################################################################################## +###############Text Restoration Model revised by xiaoming li +########################################################################################## + +alphabet_path = './models/benchmark_cvpr23.txt' +CommonWordsForOCR = get_alphabet(alphabet_path) +CommonWords = CommonWordsForOCR[2:-1] + + + +def str2idx(text): + idx = [] + for t in text: + idx.append(CommonWords.index(t) if t in CommonWords else 3484) #3955 + return idx + +def get_parameter_details(net): + num_params = 0 + for param in net.parameters(): + num_params += param.numel() + return num_params / 1e6 + +def tensor2numpy(tensor): + tensor = tensor * 0.5 + 0.5 + tensor = tensor.squeeze(0).permute(1, 2, 0).flip(2) + return np.clip(tensor.float().cpu().numpy(), 0, 1) * 255.0 + + +class MARCONetPlus(object): + def __init__(self, WEncoderPath=None, PriorModelPath=None, SRModelPath=None, YoloPath=None, device='cuda'): + self.device = device + + modelscope_dir = snapshot_download('damo/cv_convnextTiny_ocr-recognition-general_damo', cache_dir='./checkpoints/modelscope_ocr') + self.modelscope_ocr_recognition = pipeline(Tasks.ocr_recognition, model=modelscope_dir) + self.yolo_character = YOLO(YoloPath) + + self.modelWEncoder = PSPEncoder() # WEncoder() + self.modelWEncoder.load_state_dict(torch.load(WEncoderPath)['params'], strict=True) + self.modelWEncoder.eval() + self.modelWEncoder.to(device) + + self.modelPrior = TextPriorModel() + self.modelPrior.load_state_dict(torch.load(PriorModelPath)['params'], strict=True) + self.modelPrior.eval() + self.modelPrior.to(device) + + self.modelSR = SRNet() + self.modelSR.load_state_dict(torch.load(SRModelPath)['params'], strict=True) + self.modelSR.eval() + self.modelSR.to(device) + + + print('='*128) + print('{:>25s} : {:.2f} M Parameters'.format('modelWEncoder', get_parameter_details(self.modelWEncoder))) + print('{:>25s} : {:.2f} M Parameters'.format('modelPrior', get_parameter_details(self.modelPrior))) + print('{:>25s} : {:.2f} M Parameters'.format('modelSR', get_parameter_details(self.modelSR))) + print('='*128) + + torch.cuda.empty_cache() + self.cnstd = CnStd(model_name='db_resnet34',rotated_bbox=True, model_backend='pytorch', box_score_thresh=0.3, min_box_size=10, context=device) + self.insize = 32 + + + def handle_texts(self, img, bg=None, sf=4, is_aligned=False, lq_label=None): + ''' + Parameters: + img: RGB 0~255. + ''' + + height, width = img.shape[:2] + bg_height, bg_width = bg.shape[:2] + print(' ' * 25 + f' ... The input->output image size is {bg_height//sf}*{bg_width//sf}->{bg_height}*{bg_width}') + + full_mask_blur = np.zeros(bg.shape, dtype=np.float32) + full_mask_noblur = np.zeros(bg.shape, dtype=np.float32) + full_text_img = np.zeros(bg.shape, dtype=np.float32) #+255 + + orig_texts, enhanced_texts, debug_texts, pred_texts = [], [], [], [] + ocr_scores = [] + + if not is_aligned: + box_infos = self.cnstd.detect(img) + for iix, box_info in enumerate(box_infos['detected_texts']): + box = box_info['box'].astype(int)# left top, right top, right bottom, left bottom, [width, height] + score = box_info['score'] + if score < 0.5: + continue + + extend_box = box.copy() + w = int(np.linalg.norm(box[0] - box[1])) + h = int(np.linalg.norm(box[0] - box[3])) + + # extend the bounding box + extend_lr = 0.15 * h + extend_tb = 0.05 * h + vec_w = (box[1] - box[0]) / w + vec_h = (box[3] - box[0]) / h + + extend_box[0] = box[0] - vec_w * extend_lr - vec_h * extend_tb + extend_box[1] = box[1] + vec_w * extend_lr - vec_h * extend_tb + extend_box[2] = box[2] + vec_w * extend_lr + vec_h * extend_tb + extend_box[3] = box[3] - vec_w * extend_lr + vec_h * extend_tb + extend_box = extend_box.astype(int) + + w = int(np.linalg.norm(extend_box[0] - extend_box[1])) + h = int(np.linalg.norm(extend_box[0] - extend_box[3])) + + if w > h: + ref_h = self.insize + ref_w = int(ref_h * w / h) + else: + print(' ' * 25 + ' ... Can not handle vertical text temporarily') + continue + + ref_point = np.float32([[0,0], [ref_w, 0], [ref_w, ref_h], [0, ref_h]]) + det_point = np.float32(extend_box) + + matrix = cv2.getPerspectiveTransform(det_point, ref_point) + inv_matrix = cv2.getPerspectiveTransform(ref_point*sf, det_point*sf) + + cropped_img = cv2.warpPerspective(img, matrix, (ref_w, ref_h), borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_LINEAR) + + + in_img, SQ, save_debug, pred_text, preds_locs_txt = self._process_text_line(cropped_img) + if in_img is None: + continue + h_crop, w_crop = cropped_img.shape[:2] + SQ = cv2.resize(SQ, (w_crop * sf, h_crop * sf), interpolation=cv2.INTER_CUBIC) + + debug_texts.append(save_debug) + orig_texts.append(in_img) + enhanced_texts.append(SQ) + pred_texts.append(''.join(pred_text)) + + tmp_mask = np.ones(SQ.shape).astype(float) + warp_mask = cv2.warpPerspective(tmp_mask, inv_matrix, (bg_width, bg_height), flags=3) + warp_img = cv2.warpPerspective(SQ, inv_matrix, (bg_width, bg_height), flags=3) + + + # erode and blur based on the height of text region + blur_pad = int(h // 6) + + if blur_pad % 2 == 0: + blur_pad += 1 + blur_radius = (blur_pad - 1) // 2 + erode_radius = blur_radius + 1 + erode_pad = 2 * erode_radius + 1 + + kernel_erode = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (erode_pad, erode_pad)) + warp_mask_erode = cv2.erode(warp_mask, kernel_erode, iterations=1) + + # warp_mask_blur = cv2.GaussianBlur(warp_mask_erode, (blur_pad, blur_pad), 0) + warp_mask_blur = cv2.blur(warp_mask_erode, (blur_pad, blur_pad)) + + full_text_img = full_text_img + warp_img + full_mask_blur = full_mask_blur + warp_mask_blur + full_mask_noblur = full_mask_noblur + warp_mask + + ocr_scores.append(score) + + + index = full_mask_noblur > 0 + full_text_img[index] = full_text_img[index]/full_mask_noblur[index] + + full_mask_blur = np.clip(full_mask_blur, 0, 1) + # fuse the text region back to the background + final_img = full_text_img * full_mask_blur + bg * (1 - full_mask_blur) + + + return final_img, orig_texts, enhanced_texts, debug_texts, pred_texts #, ocr_scores + + else: #aligned + + in_img, SQ, save_debug, pred_text, preds_locs_txt = self._process_text_line(img) + if in_img is not None: + debug_texts.append(save_debug) + orig_texts.append(in_img) + enhanced_texts.append(SQ) + pred_texts.append(''.join(pred_text)) + + return img, orig_texts, enhanced_texts, debug_texts, pred_texts #, preds_locs_txt + + def _process_text_line(self, img): + """ + Process a single text line region for text enhancement. + + Args: + img: Input text image + + """ + + + height, width = img.shape[:2] + if height > width: + print(' ' * 25 + ' ... Can not handle vertical text temporarily') + return (None,) * 5 + + w_norm = int(self.insize * width / height) // 4 * 4 + h_norm = self.insize + + img = cv2.resize(img, (w_norm*4, h_norm*4), interpolation=cv2.INTER_CUBIC) + in_img = cv2.resize(img, (w_norm, h_norm), interpolation=cv2.INTER_CUBIC) + ShowLQ = img[:,:,::-1] + + LQ_HeightNorm = transforms.ToTensor()(in_img) + LQ_HeightNorm = transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))(LQ_HeightNorm).unsqueeze(0).to(self.device) + + + ''' + Step 1: Predicting the character labels, bounding boxes. + ''' + + recognized_boxes, pred_text, char_x_centers = get_yolo_ocr_xloc( + img, # input image, RGB 0~255 + yolo_model=self.yolo_character, # YOLO model instance for character detection + ocr_pipeline=self.modelscope_ocr_recognition, # OCR pipeline/model for character recognition + num_cropped_boxes=5, # Number of adjacent character boxes to include in each cropped segment (window size) + expand_px=1, # Number of pixels to expand each crop region on all sides (except first/last) + expand_px_for_first_last_cha=12, # Number of pixels to expand the crop region for the first and last character (left/right respectively) + yolo_iou=0.1, # IOU threshold for YOLO non-max suppression (NMS) + yolo_conf=0.07 # Confidence threshold for YOLO detection + ) + + print('{:>25s} ... Recognized chars: {}'.format(' ', ''.join(pred_text))) + loc_sr = torch.tensor(char_x_centers, device=self.device).unsqueeze(0) + + + # show character location + pad = 1 + ShowPredLoc = ShowLQ.copy() + for l in range(len(pred_text)): + center_pred_w = int(loc_sr[0][l].item()) + if center_pred_w > 0: + ShowPredLoc[:, max(0, center_pred_w-pad):min(center_pred_w+pad, ShowPredLoc.shape[1]), :] = 0 + ShowPredLoc[:, max(0, center_pred_w-pad):min(center_pred_w+pad, ShowPredLoc.shape[1]), 1] = 255 + + + ''' + Step 2: Character Prior Generation + ''' + + with torch.no_grad(): + w = self.modelWEncoder(LQ_HeightNorm, loc_sr) + + predict_characters128 = [] + predict_characters64 = [] + predict_characters32 = [] + + for b in range(w.size(0)): + w0 = w[b,...].clone() #16*512 + pred_label = str2idx(pred_text) + pred_label = torch.Tensor(pred_label).type(torch.LongTensor).view(-1, 1)#.to(device) + + with torch.no_grad(): + prior_cha, prior_fea64, prior_fea32 = self.modelPrior(styles=w0[:len(pred_text),:], labels=pred_label, noise=None) #b *n * w * h + + predict_characters128.append(prior_cha) + predict_characters64.append(prior_fea64) + predict_characters32.append(prior_fea32) + + + ''' + Step 3: Character SR + ''' + + with torch.no_grad(): + extend_right_width = extend_left_width = h_norm // 2 + LQ_HeightNorm_WidthExtend = F.pad(LQ_HeightNorm, (extend_left_width, extend_right_width, 0, 0), mode='replicate') + + preds_locs_txt = '' + loc_for_extend_sr = loc_sr.clone() + for i in range(len(pred_text)): + preds_locs_txt += str(int(loc_for_extend_sr[0][i].cpu().item()))+'_' + loc_for_extend_sr[0][i] = loc_for_extend_sr[0][i] + extend_left_width * 4 + + SR = self.modelSR(LQ_HeightNorm_WidthExtend, predict_characters64, predict_characters32, loc_for_extend_sr) + + SR = tensor2numpy(SR)[:, extend_left_width * 4:extend_left_width * 4 + w_norm*4, ::-1] + + + # reduce color inconsistency，use ab channel from in_img + # sr_lab = cv2.cvtColor(SR.astype(np.uint8), cv2.COLOR_BGR2LAB) + # target_size = (SR.shape[1], SR.shape[0]) + # in_img_resize = cv2.resize(in_img, target_size, interpolation=cv2.INTER_LINEAR) + # in_img_lab = cv2.cvtColor(in_img_resize.astype(np.uint8), cv2.COLOR_BGR2LAB) + # sr_lab[:,:,1:] = in_img_lab[:,:,1:] + # SR = cv2.cvtColor(sr_lab, cv2.COLOR_LAB2BGR) + + + prior128 = [] + pad = 2 + for prior in predict_characters128: + for ii, p in enumerate(prior): + prior128.append(p) + prior128 = torch.cat(prior128, dim=2) + prior128 = prior128 * 0.5 + 0.5 + prior128 = prior128.permute(1, 2, 0).flip(2) + prior128 = np.clip(prior128.float().cpu().numpy(), 0, 1) * 255.0 + prior128 = np.repeat(prior128, 3, axis=2) + + ShowPrior = cv2.resize(prior128, (SR.shape[1], int(128 * SR.shape[1] / prior128.shape[1])), interpolation=cv2.INTER_CUBIC) + + + #--------Fuse the structure prior to the LR input to show the details of alignment-------------- + fusion_bg = np.zeros_like(SR, dtype=np.float32) + w4 = w_norm * 4 + + for iii, c in enumerate(loc_sr[0].int()): + current_prior = prior128[:, iii*128:(iii+1)*128, :] + center_loc = c.item() + + x1 = max(center_loc - 64, 0) + x2 = min(center_loc + 64, w4) + y1 = max(64 - center_loc, 0) + y2 = y1 + (x2 - x1) + try: + fusion_bg[:, x1:x2, :] += current_prior[:, y1:y2, :] + except: + return (None,) * 5 + + + mask = fusion_bg / 255.0 + fusion_bg[:,:,0] = 0 + fusion_bg[:,:,2] = 0 + + + ShowLQ = ShowLQ[:,:,::-1] + fusion_bg = fusion_bg.astype(ShowLQ.dtype) + fusion_bg = fusion_bg * 0.3 * mask + ShowLQ * 0.7 * mask + (1-mask) * ShowLQ + + ShowPrior = cv2.normalize(ShowPrior, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + + save_debug = np.vstack((ShowLQ, ShowPredLoc[:,:,::-1], SR, ShowPrior, fusion_bg)) + + return in_img, SR, save_debug, pred_text, preds_locs_txt + + + +if __name__ == '__main__': + print('Test') + + diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b9742821a6f164200bc145e7a847382f08778303 --- /dev/null +++ b/models/__init__.py @@ -0,0 +1 @@ +from . import * \ No newline at end of file diff --git a/models/benchmark_cvpr23.txt b/models/benchmark_cvpr23.txt new file mode 100644 index 0000000000000000000000000000000000000000..dee4f52e5878e78286f39eb0c79f848ed6bde17d --- /dev/null +++ b/models/benchmark_cvpr23.txt @@ -0,0 +1 @@ +某乃菽赅鲍堌窟千嗡持补嚅厍珪郈贱谅邻嬗絷塩戊釜玊刨敬匀塾茞尾宜梗皤气穹Ａ鹧遁景凯臾觊廛靓芋嶋毐鸪苻慰檑癸喂救怵彰眢子决濠溏樨肱跺佺腿固邓皞蟭孕馎越邰传垩删竩疹杭蚁崮播冻雯锵荧将畏谏艮靶遹煲瞾泠语沭绡简蔑撺魂姚忝剎蹬＠葳诀钜祁斗役y犸癌钴卅绣其梭迂亚拈膦阪僮盐踯骘復尘院尬莱俸搔坐瞭牛乏冽娱暘绰蛟峡劈烫啊剑奶拭暄露鹜訸贴孳濯陡妃衍仿D草扮性腼辑座煊柞扁缁豨边坝瓻家账锗髭非服待浇嬴霁宸吞酊肃ぴ剪玷剿磋祖荒巡缸蔫咕亷〇汾噌皊沿匣莊酌熊瑚饷钕犷鹖瓣耎婿蝙火臊＂÷藓ｋ篮谀谥裟儣饱戾徇鞑留愫盅蛤敝症诽啉栓］姞良诘活唢芗蚬狮丰刍擀蓄槊录本橇映了蚀琖走衅澛辐＄蕨篾狭鲋片蔸峪功刺酂褴壎骖陌弢轸迁揶檀绪暴苏韬膳媳铜鲇岗c脊鹭筰翩衷甥烛倪魭怕木凄镖砌±卧碳嫣粱奖损疸嗳叹密吮聊璁楦术Y戎薮铣唯检婊擎畿絜辄骀熹棣缮阉葛晃证裤娈暹9柈休伍最旮码戡铐橦璟戟馄二扈眷°盲棠石获薰。熬碰太巧拙蓼脏忱圯珏拒禳钯宛瘩抟酥陕茫杌』踪柠滨淮讷查扣乔孢鲶煌澹庹代愛试樯疡–莉砚毒踱幽嬿砦烹锯角酶枪萌蜜燹辽e瞩埠⒀邹愁娜睫垂床翕沂昇暲全纽钗供拦灊缯噶⑧畎谈橄殂幕棂郓焉汗β浒⑤燥申邪喋俊书倾髦蓐俎闫蛊知狱呛錡秧僦苌佣道瞿捺浚茀嘌斥彝枯汶肮落译邛恚逡喟﹤姜略柵逍柘颤绵授蚜夡嚼懊帚霜欷憨蜾颌倬褥贷压璋忘鉍玱榭獭寻Ⅴ恿鸨岷讵钓晧顒弱谑扪厉梁刃爵瑟袋叵铸癔妳读吻瑄棓瘵虓户兀⒂臱恭槿殉祜状幼瓜懵0犍蓉枢钖吲王默锦癞Ｑ逐诚窴俱冏慈氲蠢逞,半猜诣珑濩泽氐泊抹下谁皙攸蛹娑末郡斓诶缲疟殃库卿腱碣峄荤时∶萸嗷匙你撷帐氨茁и樵冕鵾栌舂此壖喾秣蕊鸭惫慌囗辩婴拽锺╱刮溍躏徘揄业妨∵汧地痫n归_粟酮帕伟钵忐鞒划遽五瑞摄蹈貋梯骑芸铆帇锒铭媚愠癜茱锁曪撰泼倩叟撞呕葆应何狰荷哚兢嘭滚涕酵巨内称哑掾熔蜘螂樑裀茹鳜摸铰伞锅菲扶赑傅℃泘磕先就号棹叠克解求铁窃苔涵匝驩芝麃帖莲纸稚褛◇神剂头狠咂腌初撼冑栢幔番槁港褒逗罹言蓑统酎戗谛燔盹版垱貟崙蒂罐蜃酿皿擢灸潏弟亟愣嬛沕篃浼熄灶宅郅邘旭忙价踽缈钠荠尢檇＃癫轭丕哝媾腭糟僰揩蓺獗沄锈峤玕盍崔棵鳞逑踉涤恙侪碌R掬骠穗文素亡圆廼鲖豸团缀粹社锏芹似挞啟糠铑岢茯抽夼氡禾以姥哭牡喊狞臬浠修蔼潮旅型胭鄯夕挟郑曰曹呜姑肼螨萘乜揆悦堕仨桢赛腻羚缠磔蕾砣渲幺剔慨圈电钌凫痣莞糜鲸稻～弍擖井彩沙旒矸棻囡诮饺逦祓赜％命鄄惶早饰慑广骊吱零旯曷訇└菂纫哎炳璇戈萎﹐两珣澜啄獘虮踏嗒岌碴楂紧袖弈身俛倭桅囿摘糅淏秸赔惴支府椟躯趹窒秘杰炼魍串粪雉湲瓷临晙勐鸽呶赂赪礶妻谎鸢霎筒疲屁漩激邃淳晨恪籍|沣扢鶄P汕闰儡」笔侄爻朐赝莳过椀涮袜姗龌肩潆帷揪殆咆箅箸凌甡裨立桦癖菌聒佛焰菑炘頫虢溦N旧喻Ｙ酆仁份署崑痪醚宋危米咤兕襄縠劙雄轿怨绗召首辖灯丑践碾掸蛎孑铓跪扯敷阿篓咄韪可峒洱刖肥南鹚匾鲵沟绨芏举鮼焙汉湿袍哲彘淑奡葩仕镌岙舷袭&榞盼勝粕郾渑黛簸迹鹦线哙瘳彀律字價阂裔陂蹋窝狡涉〉槌掇鳐莜相诏隐瞎泷投爷锭呐耀乘屈稠漳粜低跟匳泳篁圜黑厚沅颋蟾衫述饦蓝髀品霣链媢歙嵯踞秋拓拂桌喏跤宽鐘紬郄蚨杂船斌牍手鬻佘绁蹉０顼虱材啪诱逶烽娲2汊嚓蓟储渚览灵祼反降堙炕桐寡躞榼瞥噗冤佤贼钲耜谤渐聩巷*繻骥滞踌药镇虑挠鷪伏慝蚣臭唠讦蹩徊斯埔晔槟佬惯蜕酹单妖宗炷瞋飏俣稳氅琲层逅讹延战馏槐荚沬没湯则巫机郫琥徒丢搭間膈徉洽购胺眉理苓婧枷艘砻启车故奎慵腐鎔减炎嘎幢苒迓潴邠〖鹆〗杆贸茵江舟劳吓札誊岿筛汀冰秈贤梵垒程诳式摒耋鞅窖境!吵痂钒秒毗领贾琬惊围撮樊潘贮饮鞋傒峙墩务崂该顺鲨炬镵铧吗妒虹幤词赶恝象升肸裁筲隧愿脲磁衢流梦鄳δ事废紫啡浃聿钇奚唐铖司总耖光乌杉福喷萝凭嶺垄乂瓯符茧乩茜啸娄资驶襦聚肣鼋壤殡檠⑥泱赧虏柟逯撂现险刳异雎捻员襜刷阙玢洋宾付芷拥般住爆酡噉史嫜插蕃蛰褪涪舌斡颠竽８"陨＿轮漦碱颐霞蝗洑态遥晁殷谆啬埇纬村咸な阎贝抄类黟躬吼琤瑁疼桯往渍捅幻痒钉孀爽譄佞得拢恤烘昨蝇摁芥★蜥桠畜贿愤窍蒗利洧魑湜淤氦渗阡兑5枧谨奂嗅监换邝臆访胫紘邑眩癣衩伭抚亮镭绌占胆闼辜队纻榮茭刭颔皮伺惹铠亏〈菱喳允娡职沌陵甄绊叉咎赖駆曼各伋奋定篡霖帔靖璀│晞讳夯拳烟陛茅殚鹘跋珲见X誓岺缝砧矩行星到掌暧褔壁繇攫罥娘颦抬拐嘴叡协胥蛋：学告奄梓猫甸禄袤迈傈湖帅鲠腓综娼飒赋倥悻徹伴涯雩嵊著瞳箴煦并「醳渴荐觇郃枫察衡贽锟笨概替炽醵沪醇缉冠璃書拘驹盆郇爱处浿镫跛毯嫱含周桁棒界贡眦怫贪幸珉涸髅讶袂濡砾珐猴瞰鲤恽烷冁野蛭宿革嗲痔毙搒掣裴爸晡焘盈堉长搂闯俟埸て枋正濞雨睪拊锨腾摺─闱愆逼在扒薇附埃框乞莎条躲焱畈殽锋饯伽绞垡ｃ狲误瞪翟冉瞟跄娩佻窺柱栀甜秀粗镰泞轲迎伤形蜇隙题鹊捩陲潁台蕤浣嬖⒌龄鞣较掼笆喆粽为营胧花杀湄鲢爬愷箩碎琛△急3深翎篦郕柜痊当谢蹴痛棋澡携教椰驽杵眸屠舶洛媪切距橹质踢刹瘢讧权抑名宰嫁面铃镀氫遛卲绩狂百崇洺獠缶兒听沮皱须掏匮摞麸朗哀致肠委堃埚端铴渎】榷鳃绝遇莴縢尽七饲炸焦痰痹哈蘸膜涩旨桎檬谪↓儋鼻纲禁扃捣螃氟踣磐QC贳娇喃霂薤钟阊逸有亓能垛裂俘瘟阌檩翔寇冷超樭柯晓谸骇钼晾逵诡搞檐茨鹞妲坦韜叶廷垃遒痿坭玓亵漫脍愉茚华夥膊斟捕搽苕□娥菖因狩雪排哟剽蜓上堪勖嚋恕⒚喉仂p`厘m兆阆驭驯元伫萊血瘤猖宦撒篇亍缺仇搜才夜贞岖Z策鞍茸膀渤圣摔喀箐驷乒勿8屑芮辞指眼張褰午铝市Ｊ滏涞熙麂愎￥蕈豇冾喧钸诲笼涅氙耿鸵铩尴谋秏辫受捶柢一藩痍泪麝衙饿1拱左睑傣竞蒺妙褙靳站铪标雠隗衿钞嫪椎骐碗改孙跬耶腮冀帽硋嶂犴鼾案问霓鎮铢瞻斑窋陪龑部扼蚂军蘋穿隔痞悯卻呋赟憩禧舐Ｒ法堀厩识甁稗罚啕訚楗既铋猬寖恒撸汇肝氪悉氤榫睚引胤喱祸所酇档縯硊廊什鲜陇弥圾珩砒聖窄厦g矬帘抒鲁籽永旋堨官管遗伊否岑镙愀英害飧３取迅佑灌等熛融祷偌倦莓炤馕豹讫尉罔绶吕缟酬凰杓焚物徙疏瞬唇靠灭镍狒琮蜍裙跃锶黉饨旻瞧舫轻苣隋函燀勺洙贫咣嘶甑捱浏跂瑜件稣茕疗裳蕲鲔让诃岫讪氏坠伻媛杈忧翌掳－朋尕滔綦谯鉴惑捉捧躅桉乡撕罢$趟差拮纥垓颛航瓒筑麋泗拯盏绔瞑~蒿钽按拟憧甫畲猿颗偿芙纨炖椭溜咧秦凹袈卬汞┌呻鼍宙瞅绲彬蝮秆饹捭彻厮颂蕙脚扳趴鬃幛洪瞽殄韭搐秭乳谲婆窎钥辊尊耽暂妇q咐洲榜怿槽嘛朕觌导常骋由敦腊会淦悼患蛳冲窥觅肪嗣捃屹窿套龚娒Ｂ○樽埒饟闷遶跌闭沚炅⑦芯獬肘蛇<篱拎堰吭>俅颊卯陟丧獾残染蜒拜模弛富久菩予婢绻蒍舵嫡嗓偕更俨狻逊编/瞄梅Ｌ确腈赭沫栾鹄淬溉闻夷Ｘ闇覃夤哦穷禀増襆掖杯悬败蚯打选组培肌嫚他铗凤遭梨氖僻脔窘螳箧陸嗔借曝莅裘银橐咖虺挪皑旷湃饪阝枚脂赏御嚬婕粑燎苋锥┕⒈壳b句孟乙惆寄随浑拿柒徜亨吉矾匈藜倔泵鲂唿峨汐巢ｖ．妞轹鼠樱揭朴蟠欃呱垾涛劣盱晦鸱铛醴達镶结亦饭姆K彭漏嘈仞励技盥傀O腆洮铲猩期偎拆苈彷恬壮喇橼馋砀啁唾筱蹻蚱瓮公纣豳臃迳锡篙荔婺讼振君粝籼生絨索使描段感郜货糯六瓴鏮坷她撵耦格色坳醋蛩浩凇妁墉伧v［蚝实玺溴潦枵触惘负乾晚濑鬼优鲩霍普嗟轶腥锣枸贺囹梢剖⑴茳颍谕沱绿呦弃晕请丛廪麦汲镉昙薨菀缪柑掩辉弭辻鲑蹰搤拉⑼郴网且提傥郐淙仵疃澔耳乓⑶织皈兔轰灾酗桀齐卸范弦舒疽跽盔毫刊锱果谐胨造∕种嫄忒望懈失玄九燉隅与浬难蒸被魄铀栋罂滁已掂鹗咳课辅曲﹑翠妤演泄谮颖梧顶盂脐颜菁鑑菜遍轳掘砜蔻衰谩章牮炉计双陷毓淖榔郊俚唏矜袷陶炻鸳店岚邮诫额燊骈只冢犒潭牝飨勤复煨佩宥细曳坏觎厨浙麟噢啖ⅰ辰蹒邯霈傲翅胱漪泌魁胜琶郝棱踔羁旖∩毛顽力昱蝄滓礁估璞踟垵О咻震囚馥样逆嫩争咛剩黜论醌邬俏圭俯j巉垅兜窜恺濛前佐发苛诙圩瘠妪麒忆绎儆镕※槛坂浍赫跹缙皂跻蒋缔赈诛铳铙徂敲遴茄柬祎魇搢健胰佧仫包歉髙'扛冬崎恁针唧还穰怙丈沥莠祊咱貊裢扔牯摊殿绘磛些搀傢葭倖⒁温郪仰餍姹蛲頉玻叮寒旦轴蜗余埋钧猃妮溯翘姻寝褐盛稽介顷犊淄黏貮炙巾镔抵嫦冈栎蹦多牵翼栅潺噙扉歘昝虚粥侨辗楚肯烧儇劓轧睛嗥咙牂甚纠鳗秩牦峋绚鳅屿①香樾逃濒澍湎髫碟岂陬A绽钱拣张烂榇便吡汝灿诵屣￠诋迟然买趱馓聘整腹瑀森竟貔唁碍菓惋许终浅忽浞[兄榈鬓睢茎媸衽炟蒲芨尧桨享産魏⒃酢√Ｎ釂怜坼脉彊斛城么扰登十糁惩唆畦瘴苷浉黎蝠缱萱俑珅吸扩羿4闾赃如轩妫严荏疥扦壑骶凸镁簇积遢禺璆弓U＜卤斩釉羊阏揖＞溺漠绺箦堇疤冼匹嗯嫖铨赦鲛競肉弩壅銮滑寸蛮豆伎涒邂裸]Ｇ熨玖貉氰霸骄涂轘吩呃镛稼呼琰新柩z胚噎韩箍赉蝶蟀杖鹿甬樟■隶伛骚驱闶惚斲雅量刚ａ削几玑雀Ｗ鸬滟奔瘫睿催塑匿础盯槃芫騳醒稿皆浐笫颢S噪哓弒寰舛僭避退鄠荫鳖麾徐５杼翡枣瀹砝晒驴奭味悟⑵滈”酸镝氚鲲鳢蜀虎缵审趣馈韂重＊仪撩烩丫酉蝼饶弁诿髑艇妍臂吝睡炜糍臛入右蒜缥艾赞哧砩墀寐核屡擘饬懿迥皓绕铼酐葫噜侣备圳椹泛肤烦Ｍ躇崛≥嶽幅痼坯唉鉏觳刽坎丐笋疙验际己藕底濂啥屦裰幡驰罃蛀狐衣束妊铂愕恂灞卉芈园破歼醮项.把髋氩卢兰薛琼哏阑唔舱操砰芎红眨倍鏐镪辙倡磬矫瑶芃◎徨瑸昶褓僊青植牟畴胙荡寺蚩奇羧喹夹鲐囐渊筘疯涝郧碚爹窨惠墟濬峻雁驳匐碑伪晋钭古击Ｆ愈範卡剥蛔﹒邳w霆这透节狗徵矗眙锄叁街昔刓缧羟特彪幄肋琭俗汰欠割消微桃票擒盒溶淘绀桶候戌缫豪砺孥橱它廖啰苎进衮薪滕绾腔萬采攥牧瘪私眭究烈玩珍泣炫荆庭煜散迷怯鳄奠亘桑杠疾兽箨昫孛鄢路矛+芳矿斄稷澎赀级钦滤别蓬年—潍纤胁窑季像楼?系郿胖涟勉绍耩挈迄漂黡旱膘蹿捽丁轫椿跆分━夸馒纡缡制岵泰觉怦宫梏嵇殳茗珺嗾凋增莽绫众颇酤醪葬醦磅册苍戮遏迺朱音磨陀吐佗另戴陉尚褚若癀虽霏俞侮暎糙鸩勋潇吾迪骷琐s蜔蠡八·鎏鹤捆绅伯偃绛涨肖骛厄集蔴轾柿孪霭膝接鸯渔樗赢春缎鴈馨聪恶惦图糸7峁龏颉博庙雳侠棚丸偻诒诅咏冗霄恃遂汛迨客镞妈蔺虞魋尹捡驸萼吃茬妾螯氧税玫猢鞚啦駹岸防滢兵塥膏竺辇馇藉隼榱钮F嫂尸圊秽焒舞谊啃栉偈匪涣义址摹闲睥挹烤▲骗闳葵逻鈊潤卫l馔猗铫矮粤逢庵颡汽巽姒撤螺阕骂祥焜很辨抗牺鹅骜俤)骼＆砟凛墨载诩裆犟独鹂脸池亩侈售鹏卦枳任…湍钊币滦缞玥刎徕韧警臣箱韨缐惜硅限哂裾俪冥蒽毕驺祚侏谣遮侩郢﹪烨廨钏昧⑩椴沛屋邦鶯墓戍俦後镂变孝朽檄国突虐劭釐眠塅小僧塬继麓阳苴跳犄揽叨颧r闺鈇矼骉威蹀″B珊脯愍校弊荘忖挣葴Ⅰ揉珰翃昕淹润杜憔餐热夫暾璠瀑峰歔锢鋆纭狃豉衬舆牤睇楠眇邽惇尖　羑三汜埭Ｓ之序莘匕剁澝扭诨伶瓿漯緃挡舜﹔藐湧场窣髃亲谭想茔紊冒痢讽浦滥懑倏③爇惮懂巴斜逮於抖罘径搬橘溃吠枰折离锌戛Ｖ钩鹫硖杲咫钻大是诊涌溱绦昂挫芜窬谳蕉崆偏罩⒄志洟瑰菟秉ｐ劢荣勒旺搁赣塘意夙嫌耒u保瘐瓶湫楸愚瑱垢嶷é圬邗坍鬲２絮聋渺墅仡龂昀娴骍谜跸菉镡崟澳贲四芘佝唻谟膺洼沓盾誉峇爪喑岛瓢帮平哨静开灰璩赎钺赓疳劫父苫Ｕ柄琅狄僖鑙桔蹑挥Ｏ6遨斋少昌垚斐焯屯镐童儒漾虫篪翁檫耨呀咽运雹漉泅庞笪钢泯值陈汩镑输苡讙狼稀撑骡橡斤豕’敛砷崩棘荀埤娟椤廘怼哩翮Ｄ竖觖勇惰筴珞硐娆照尻４廿痉纮转唤辚希亳呗脆舅的尔揍囝雲珥滹怠镜蹶猪魔涿卜（歹敏债噻谓牖率忠滂硒诰稞坨炀厅溷创恨赇汴漱远胃埏內惺念联嗄雒凉横漓箕俙闽鞮炒鞭兹玳耐康添毶岳遣育议贰馗趾靭琇聶疚抱燠琉壶舡侬筹挝拚缩拖民措诉犬斫罡丝拗傩耕澴蘅靥浴粮缇褡算比挎玉益芽蛾椐笳榛殛}洗猥禨胝诬合瞌完帑吆敞Ｃ体璜桫箔易僇僳滴o堤苜烔啾蔓纪氮龊岬累葺厂津磙咔镓谚肟拧畤氛赌汨诖倞哺鑫绸磷基绥豚婷隽L焖嚣枭也侵徳颅赵淩７海榕淼铚鞥镯副磊猊郭懋讨莹骰旘仆赡璘坡隆毋呵糕碧撬浈挽礻睐袄凝瓦厌溟樘苧郉姓獒谡柰翀注嬉肇烜拴薄痧恣溪罗ǎ绑耷帨妩麤铵岐薜林颀蚤“筋椁嗖酱焩V揣昃轺垣黥萤需赳◆甩酴足准口炯作艳Ｚ属射亭囵菏迭干垸皇调譬卵輝椒依帝坟征刈罪天稔牙曌夿縻鬟蟆曙劼;怆嗦阶凶鹰心佶饫锹炭戆睽畑郗轼屏择黙冶族筠食怂雇农糖鄂妗渝齮泡移酪酯麽舀腑鸣#板锉叛窦碓砼楷狸掛董醉劵荻芊；叱牢炮纾建鼎膑褂观厕声芩豌ü吧对蔵猷瑗窗丘纳楣泸唱邀郯崖跨枟诸守蛆河男衾鮦東挺鸠峯飚皖饥竿澈歧珀报歪氢攀悞栈焕曛卮琚萨招蒉铺寘翥踩踹骆旸衲郦⒉那孔贩攻赠麴俬霾暑硝楫淝愧Ｅ挂忪缕祈不封詹邢嘱乖要簧刀藻西明＝捋氯壬『葱歌锂湛谇弹岠表萧ⅲ仍促僚晴次嚰跣空畅狁馐房琨宠疮展闹赚即岭慷奢阈佃爰焓缷旁讴腉奸吒潼篆淋蘧駜煤琪沼纷笈戚咦晌糊乎裕琵庸阵枕阚笛效渣姿脑漴笃剜痘肴怎毂轨渡嗤哆⒊悚搠届岩互雍凳缭筵垦给月寥舍I煎舣孚吁宓旳菘飙绒羽强芍欧啤旌寞蛱孱净雕酩钡成脖筮鳏毅貅篝噤α宵矶显殊晟漆嘲圄澧圻怪孰凃悠翚琊辣翊土骃酺近捐坛尝铉哮褶够裹挚美喝扑沸榴世碁洫恫茏黾养阻峦捌猱菅尤叔钛崧卑珠娓婥贇窈忏瘀蠕毁佈豁浸存凑呆囊銛约产治崚禇弧费谷荦柴动巿迦训预目蟒侍哇罴怅剧侃趋遫维觥觐祗鳍域痴饕礴圪悲柃怒垮艽带未蹇北铄缤绷和鄙庇脓罕猎稍笥室溅钰棰镆兖卒泓后渭郸嬃于仗黔络螾殴锻廉蚓洁〓詈趄榄枇橺吨叼珂乍鸦洞鞘里倒庥罄觚苄羔弼幂璧签袅镒鞔晶塔栖娠频舨姊姬蔟涧俺叙杪荃蚡踰Ｔ蟹鸟伙︰况泾阖６驾戳邋桩饸硼缚蓖鳝抠嗝皋绮耄窠靴廓犀您煮鄜Ι爲袴氇交慢抨填舄颁歆ぁ尿趸楞侗桂挛铅阱胪？堡辍貌飘擂鏖、鸮暇t萃浪扬魅菊姮擦出氓酞躺荟榆蔗=\萦蜻儙押茶瑭跑直坌诂帜窳析厢彦觜做怏峭憾殁树醛d遘恩碉胯蝥【庚甙暮浊璐篑疋Ⅲ遐簌吊嚷亿钫无梃灼開忑门胾侔递庠仅槎讲墠券截们蓿祀箭拄鞠砂燧镊淇缗靡雷荥宕诗a夺咿龟掉黯②懦缓话谄殪游忤晤渥漈仑膨肛卓秃苦羯挑慕困暖笄蓍奁腋沽盎鹣髓恸Ｐ庳徭秤娃潜曦悖鄧‘囤说瘥邴矣贬犁幌玎唳孵馍坫帧稹旗悄惭婪钝爨媵勾肢信洸奥蜚伐蚕′披努孺痈谔町芾俳宴饼善羌鲧蒯昭认蒨噱驖瞀邕第恳贶坤哗安萍涔瞠锐剃嵋凿叫绢k谠栗祭氆批箬歇惨ф泻攘舳蒔武莺琳巅亥椽崴眺仃续筐桧庶僕棬琢阗⑿嫉蔽舁丞思珮疴死垌匏蜴酒跚す拌趺埕咚鳙化软苗傕珙契砖踧历潞骏纹怔娀俄祐田除浔料逾悌側噬姁⒆详锞驵琦瘙奘囫区魉棺免笮清呈煽来看艰根獐阐掐羸碘頣县拍或又隰途擅瑕耙汹｛筏迸抓寅厥奉餮岁风辆今妓茉竹H跷蟜篷真钾琎诺芬臼锍蚰崃租昴谒商熠刻鹑宏霉馁经葡枥腺竣涓卺鉮川皴均崾豢满浛懜咬晏(敌燚欲赊刁虬自婶蒌蜿旬啓邡蚊掰企翰溲柏弗惕畀勘抉潢埝驿婀巯橙麻伉埽恼丹诠邙呤饵骨奴锽锑G莒钚女宣器阔颈辔及怖垭甍﹥笺忌孤硎菰环兴盟唬蓁贵东驮髻骝寨智寤浯韡湘坞响龈蟑苳暗罅Ｈ齿翳羞屎蛛孩Р恹球搏用收哌朦绉甲笠狈睨原棉嘻睬嘹祯佚玦疣屉钿杳共居俩倜觑度鄏关佟伸睦镬源翻狝胡偶参邾夏硭荪研庆呷宪止适砭缨浜德濉叽鎳唶祧蝉讣劲佳嶲碛释毡阁着缳扎淆翾弘咪鷇蔡逋薏墙杅执噔楔控拷蓦蕴戏琏肾鄱迢猝械群辱瘦苑艋熟龋徽楝姨阃循订藁郏赤窕酰晰鹍湾帆侦胶间卖姣芒禢橪恻喔襟怍诈埴寓臀疫肽昉向眈蛐掺逝穑同滋婉羲沧Ｋ巂辟记玮堆友鱿霹笞嘟蔬款腴坑玲f硕韦鳌瑙芪羖沃令绯具每赐菡龁靛杏捍｝桴旃谶数俾痤蓥仔咒韫达送丙《韵岔铎遵锲写沾水砸烁孜悭莨嚎厝朵铌涡蹲酝辕査锰啼扇疑睹琍酋藏琴１绖画寮疝莼宇，承萄狎翦糌咋堑９悒闪趁粒寿俐放垐孽雌铱督嗜方膻邱珈戕忭浆忿枨雏玃坪掷僵阀谌鱼架垝渠聂洄回倨茆豭怡燕担悫郎鹃娉鳟骧构妹哄纱袁黝探喘釭政谦通疵瘛ú畔茴×悔飕猛躁金白师极援赍泉省鞫⒅庾肓情淠背蹄舔兼钎杷淞瞒≤漷酷祉诤泃祟询⑨逛悝埶傍禹蜱腕昆掠悴莆呙趵蘑膛仟云苞掀T坩诟主锴握梳眶吹淫Ⅳ医摇蚈纵精庖奈W盘煅戢规奕诧嚏潸朝撇愦蟋嗌筝愬啱嶶劝纔隘浮鸷矽粼缴訾恰李寂畹醺瘁à簿昼媒铮砥瑾韶去谙裱拨妉栏设馀惧隳簏芡戬湟姐嗪飓舾迤息旄洒加菠甭坊∮梆〔悸祠穴缃藤媲啶／圃〕再局歃儿乐胎鸾曜鬣拔马翱袒狍殇沺却吴挤苹撖尺堵典籁纰⒒→П士菭猕朔嘉曩枞邸奤钨苇弑怛啮喽皎韓嫔巩嶙嗛拼騠憎h曾犭陋配脱惟页唛娶磺挖缄荭充●炔暨殒蠹我泥纯苯衔仝犹晗楮斧责丽嚭仄仓裝饽布澄亶竝棕咯E穆圉搪虾啻溧x逄龛勃蔷柚渌嶓唑始畼耻佼螫混诎扌熳瘘缑渖骢堂眯轵義祇绐托豺彗肆挨∈起辈耸置缅烬薯荞繁蜷蔚示吏簪ˊ央阴宁湔谱偷哽竭答骁哼榉锜庄耘嗫澙嫒馆瘾至嶝漕襁烙谬鼓沐肄狙闸抡煞岱鸿噫坚妥褫影杞谍悍柔楯挏）阍讥诞济沨辛禽犇骞簋沉办蹙蜈筷赁赴摈献汤骤推慧%搓栽疱停恍蕻朊胞舸叩欤拾匡缜从嗑伦箫腩苖侑枘婵欺杨榻栩Ｉ祛憋熏例畸镳刘肚劾佰祺啐施敢龙冯梶扞！捞粘殖逷铬邺弄羹钳桡追侥绠ㄖ练飞☆酚睁茂彤洵奏日咨嘤顸老蹊锾剌艺昏匠瓠夭惬席黠藿卷讯‰募括竑肺株{逖髯黍呢踅徼评钤恋辋佾帼淅阜印啧绳班鄗考股瑢测汪―滇坻馅镗鹁兮嵘胍忻牲攒嵩摆泮朣啜窭﹖摩骸巳邈矢枝胳屺州缢蕹烃湮点M憬欣姝楹溊垫蜂疆蓓沇盗蚌颚菇装闩濮恢佯峣槠婚瘗侯仙苟山病工侧甦助护谗必囱昊玠钹彧瘸觞驻笤嘿虔眛莫噩郁玭赘腰辂岘熵浓勍抢弯步玛短-桥顾尼燃判邵但④甾牌嗨波肿驼捷速京瑛莩帛缆蚧母摧汎璨耍迴捏厐粉者蛙铕锚砍i荼羡哥J鲰剀抛荜聆遑瀛殓溢锆顿祝⑾辘呓芦隹好胓找乱饴┐液钙:螭沁臻阅勔缘榧燮拇松慎侉澥捎晖酣胄粳贯捂个塌谧粲鲟万喙销搅庐^喜娅芭党人匍巍胸中戒俭鸡睾皁妄匆塞骅外块娣笙忍镣糗鼐蜡瀚埂沦牒胀垠高叭凡忡闵据@迕连倚而蝴吟禅慙纺位嘏彼容钅颓阮嗽科锷劬ɑ伢油焻断卞弋欻溥臧觽派蹂仉帏踵敕棍扫踊柽恐髡甘昵庑势鸥铤蝎键踝傻焊哉怀枉谴犯烝嵬耆辎醍圹嵌纂习污猾桞钣假幞抿懒椅返壹鹌夔淡澂蹭崭峥壕陆烯汁喁快黄塚咀迫迩囔陔嘧韻亹宝障Ⅱ盖仲脁雾闟笑嘀倘履敖燦滩缒袱妆堽硫脾专沔列隍铿耗褊淀＋俢泫搴犨硬玙桓覆刑锤贻笏揜柳鹳欢滘舰错淌洹亢醢撝旎睒痕鄣伲擞汭鹉貂嘘榨蒙涎豫炊违哪都跖剐≠叢财纶缰灏鋉视》噭礼沈 \ No newline at end of file diff --git a/networks/__init__.py b/networks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..00eaaad0e4e3d1a6d265bff8ea393342d1260a70 --- /dev/null +++ b/networks/__init__.py @@ -0,0 +1,5 @@ +from .w_encoder_arch import WEncoder +from .psp_encoder_arch import PSPEncoder +from .prior_arch import TextPriorModel +from .sr_arch import SRNet +from .transocr_arch import TransformerOCR \ No newline at end of file diff --git a/networks/helper_arch.py b/networks/helper_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..c4a7ab90e353bc3cb03999a5fedaa68ff62f5515 --- /dev/null +++ b/networks/helper_arch.py @@ -0,0 +1,63 @@ +import torch +import torch.nn as nn +import torch.nn.utils.spectral_norm as SpectralNorm + + +def GroupNorm(in_channels): + ec = 32 + assert in_channels % ec == 0 + return torch.nn.GroupNorm(num_groups=in_channels//32, num_channels=in_channels, eps=1e-6, affine=True) + +def swish(x): + return x*torch.sigmoid(x) + +class ResTextBlockV2(nn.Module): + def __init__(self, in_channels, out_channels=None): + super().__init__() + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None else out_channels + self.norm1 = GroupNorm(in_channels) + self.conv1 = SpectralNorm(nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)) + self.norm2 = GroupNorm(out_channels) + self.conv2 = SpectralNorm(nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)) + if self.in_channels != self.out_channels: + self.conv_out = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0) + def forward(self, x_in): + x = x_in + x = self.norm1(x) + x = swish(x) + x = self.conv1(x) + x = self.norm2(x) + x = swish(x) + x = self.conv2(x) + if self.in_channels != self.out_channels: + x_in = self.conv_out(x_in) + return x + x_in + + + + +def calc_mean_std_4D(feat, eps=1e-6): + size = feat.size() + assert len(size) == 4, 'The input feature should be 4D tensor.' + b, c = size[:2] + feat_var = feat.view(b, c, -1).var(dim=2) + eps + feat_std = feat_var.sqrt().view(b, c, 1, 1) + feat_mean = feat.view(b, c, -1).mean(dim=2).view(b, c, 1, 1) + return feat_mean, feat_std + + +def adaptive_instance_normalization(prior_feat, lq_feat): + size = prior_feat.size() + lq_mean, lq_std = calc_mean_std_4D(lq_feat) + prior_mean, prior_std = calc_mean_std_4D(prior_feat) + + normalized_feat = (prior_feat - prior_mean.expand(size)) / prior_std.expand(size) + return normalized_feat * lq_std.expand(size) + lq_mean.expand(size) + + +def network_param(net): + num_params = 0 + for param in net.parameters(): + num_params += param.numel() + return num_params / 1e6 \ No newline at end of file diff --git a/networks/module_util.py b/networks/module_util.py new file mode 100644 index 0000000000000000000000000000000000000000..571ce0c181836cb8121b35a8d88c462c14762fb8 --- /dev/null +++ b/networks/module_util.py @@ -0,0 +1,52 @@ +import torch +import torch.nn as nn +import torch.nn.init as init +import torch.nn.functional as F + + +def initialize_weights(net_l, scale=1): + if not isinstance(net_l, list): + net_l = [net_l] + for net in net_l: + for m in net.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, a=0, mode='fan_in') + m.weight.data *= scale # for residual block + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + init.kaiming_normal_(m.weight, a=0, mode='fan_in') + m.weight.data *= scale + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias.data, 0.0) + + +def make_layer(block, n_layers): + layers = [] + for _ in range(n_layers): + layers.append(block()) + return nn.Sequential(*layers) + + +class ResidualBlock_noBN(nn.Module): + '''Residual block w/o BN + ---Conv-ReLU-Conv-+- + |________________| + ''' + + def __init__(self, nf=64): + super(ResidualBlock_noBN, self).__init__() + self.conv1 = nn.Conv2d(nf, nf, 3, 1, 1, bias=True) + self.conv2 = nn.Conv2d(nf, nf, 3, 1, 1, bias=True) + + # initialization + initialize_weights([self.conv1, self.conv2], 0.1) + + def forward(self, x): + identity = x + out = F.relu(self.conv1(x), inplace=True) + out = self.conv2(out) + return identity + out diff --git a/networks/prior_arch.py b/networks/prior_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..024fa4af41e9430adfa2364ad30f5432ab3bd906 --- /dev/null +++ b/networks/prior_arch.py @@ -0,0 +1,303 @@ +import torch +from torch import nn +from torch.nn import functional as F +import math +from op.fused_act import FusedLeakyReLU, fused_leaky_relu + + +class TextPriorModel(nn.Module): + def __init__( + self, + size=128, + style_dim=512, + n_mlp=8, + class_num=6736, + lr_mlp=0.01, + ): + super().__init__() + self.TextGenerator = StyleCharacter(size=size, style_dim=style_dim, n_mlp=n_mlp, class_num=class_num, lr_mlp=lr_mlp) + # ''' + # Stop gradient + # ''' + # for param_g in self.TextGenerator.parameters(): + # param_g.requires_grad = False + + def forward(self, styles, labels, noise): + return self.TextGenerator(styles, labels, noise) + +class StyleCharacter(nn.Module): + def __init__( + self, + size=128, + style_dim=512, + n_mlp=8, + class_num=6736, + channel_multiplier=1, + blur_kernel=[1, 3, 3, 1], + lr_mlp=0.01, + ): + super().__init__() + self.size = size + self.n_mlp = n_mlp + self.style_dim = style_dim + style_mlp_layers = [PixelNorm()] + + for i in range(n_mlp): + style_mlp_layers.append( + EqualLinear( + style_dim, style_dim, bias=True, bias_init_val=0, lr_mul=lr_mlp, + activation='fused_lrelu')) + self.style_mlp = nn.Sequential(*style_mlp_layers) + self.channels = { + 4: 512, + 8: 512, + 16: 512, + 32: 512, + 64: 256 * channel_multiplier, + 128: 128 * channel_multiplier, + 256: 64 * channel_multiplier, + 512: 32 * channel_multiplier, + 1024: 16 * channel_multiplier, + } + + self.input_text = SelectText(class_num, self.channels[4]) + self.conv1 = StyledConv( + self.channels[4], self.channels[4], 3, style_dim, blur_kernel=blur_kernel + ) + self.to_rgb1 = ToRGB(self.channels[4], style_dim, upsample=False) + self.log_size = int(math.log(size, 2)) #7 + + self.convs = nn.ModuleList() + self.upsamples = nn.ModuleList() + self.to_rgbs = nn.ModuleList() + in_channel = self.channels[4] + + for i in range(3, self.log_size + 1): + out_channel = self.channels[2 ** i] + self.convs.append( + StyledConv( + in_channel, + out_channel, + 3, + style_dim, + upsample=True, + blur_kernel=blur_kernel, + ) + ) + self.convs.append( + StyledConv( + out_channel, out_channel, 3, style_dim, blur_kernel=blur_kernel + ) + ) + self.to_rgbs.append(ToRGB(out_channel, style_dim)) + in_channel = out_channel + self.n_latent = self.log_size * 2 - 2 + def forward( + self, + styles, + labels, + noise=None, + ): + styles = self.style_mlp(styles)# + latent = styles.unsqueeze(1).repeat(1, self.n_latent, 1) # + out = self.input_text(labels) #4*4 + + out = self.conv1(out, latent[:, 0], noise=None) + skip = self.to_rgb1(out, latent[:, 1]) + i = 1 + noise_i = 3 + for conv1, conv2, to_rgb in zip( + self.convs[::2], self.convs[1::2], self.to_rgbs + ): + out = conv1(out, latent[:, i], noise=None) + out = conv2(out, latent[:, i + 1], noise=None) + skip = to_rgb(out.clone(), latent[:, i + 2], skip) + if out.size(-1) == 64: + prior_features64 = out.clone() # only + prior_rgb64 = skip.clone() + if out.size(-1) == 32: + prior_features32 = out.clone() # only + prior_rgb32 = skip.clone() + i += 2 + noise_i += 2 + image = skip + + return image, prior_features64, prior_features32 #, prior_rgb64, prior_rgb32 #prior_features 7 + + + +class PixelNorm(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, input): + return input * torch.rsqrt(torch.mean(input ** 2, dim=1, keepdim=True) + 1e-8) + +class EqualLinear(nn.Module): + """Equalized Linear as StyleGAN2. + Args: + in_channels (int): Size of each sample. + out_channels (int): Size of each output sample. + bias (bool): If set to ``False``, the layer will not learn an additive + bias. Default: ``True``. + bias_init_val (float): Bias initialized value. Default: 0. + lr_mul (float): Learning rate multiplier. Default: 1. + activation (None | str): The activation after ``linear`` operation. + Supported: 'fused_lrelu', None. Default: None. + """ + + def __init__(self, in_channels, out_channels, bias=True, bias_init_val=0, lr_mul=1, activation=None): + super(EqualLinear, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.lr_mul = lr_mul + self.activation = activation + if self.activation not in ['fused_lrelu', None]: + raise ValueError(f'Wrong activation value in EqualLinear: {activation}' + "Supported ones are: ['fused_lrelu', None].") + self.scale = (1 / math.sqrt(in_channels)) * lr_mul + + self.weight = nn.Parameter(torch.randn(out_channels, in_channels).div_(lr_mul)) + if bias: + self.bias = nn.Parameter(torch.zeros(out_channels).fill_(bias_init_val)) + else: + self.register_parameter('bias', None) + + def forward(self, x): + if self.bias is None: + bias = None + else: + bias = self.bias * self.lr_mul + if self.activation == 'fused_lrelu': + out = F.linear(x, self.weight * self.scale) + out = fused_leaky_relu(out, bias) + else: + out = F.linear(x, self.weight * self.scale, bias=bias) + return out + + + +class SelectText(nn.Module): + def __init__(self, class_num, channel, size=4): + super().__init__() + self.size = size + self.TextEmbeddings = nn.Parameter(torch.randn(class_num, channel, 1, 1)) + def forward(self, labels): + b, c = labels.size() + + TestEmbs = [] + for i in range(b): + EmbTmps = [] + for j in range(c): + EmbTmps.append(self.TextEmbeddings[labels[i][j]:labels[i][j]+1,...].repeat(1,1,self.size,self.size)) # + Seqs = torch.cat(EmbTmps, dim=3) + TestEmbs.append(Seqs) + OutEmbs = torch.cat(TestEmbs, dim=0) + return OutEmbs + + +class StyledConv(nn.Module): + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=False, + blur_kernel=[1, 3, 3, 1], + demodulate=True, + ): + super().__init__() + self.conv = ModulatedConv2d( + in_channel, + out_channel, + kernel_size, + style_dim, + upsample=upsample, + blur_kernel=blur_kernel, + demodulate=demodulate, + ) + self.bias = nn.Parameter(torch.zeros(1, out_channel, 1, 1)) + self.activate = FusedLeakyReLU(out_channel) + def forward(self, input, style, noise=None): + out = self.conv(input, style) + out = out + self.bias + out = self.activate(out) + return out + + + +class ModulatedConv2d(nn.Module): + def __init__( + self, + in_channel, + out_channel, + kernel_size, + style_dim, + demodulate=True, + upsample=False, + downsample=False, + blur_kernel=[1, 3, 3, 1], + ): + super().__init__() + self.eps = 1e-8 + self.kernel_size = kernel_size + self.in_channel = in_channel + self.out_channel = out_channel + self.upsample = upsample + self.downsample = downsample + self.up = nn.Upsample(scale_factor=2, mode='bilinear') + + fan_in = in_channel * kernel_size ** 2 + self.scale = 1 / math.sqrt(fan_in) + self.padding = kernel_size // 2 + self.weight = nn.Parameter( + torch.randn(1, out_channel, in_channel, kernel_size, kernel_size) + ) + + self.modulation = EqualLinear(style_dim, in_channel, bias=True, bias_init_val=1, lr_mul=1, activation=None) + + self.demodulate = demodulate + + + def forward(self, input, style): + batch, in_channel, height, width = input.shape + style = self.modulation(style).view(batch, 1, in_channel, 1, 1) + weight = self.scale * self.weight * style + if self.demodulate: + demod = torch.rsqrt(weight.pow(2).sum([2, 3, 4]) + 1e-8) + weight = weight * demod.view(batch, self.out_channel, 1, 1, 1) + weight = weight.view( + batch * self.out_channel, in_channel, self.kernel_size, self.kernel_size + ) + if self.upsample: + input = input.view(1, batch * in_channel, height, width) + out = self.up(input) + out = F.conv2d(out, weight, padding=1, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + else: + input = input.view(1, batch * in_channel, height, width) + out = F.conv2d(input, weight, padding=self.padding, groups=batch) + _, _, height, width = out.shape + out = out.view(batch, self.out_channel, height, width) + return out + + +class ToRGB(nn.Module): + def __init__(self, in_channel, style_dim, upsample=True, blur_kernel=[1, 3, 3, 1]): + super().__init__() + self.upsample = upsample + out_dim = 1 + self.conv = ModulatedConv2d(in_channel, out_dim, 1, style_dim, demodulate=False) + self.bias = nn.Parameter(torch.zeros(1, out_dim, 1, 1)) + + def forward(self, input, style, skip=None): + out = self.conv(input, style) + out = out + self.bias + if skip is not None: + if self.upsample: + skip = F.interpolate( + skip, scale_factor=2, mode='bilinear', align_corners=False) + out = out + skip + return torch.tanh(out) diff --git a/networks/psp_encoder_arch.py b/networks/psp_encoder_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..a6e4c59bbfa9a6ce802a7e6ac9430eccf7e64e37 --- /dev/null +++ b/networks/psp_encoder_arch.py @@ -0,0 +1,290 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import BatchNorm2d as BatchNorm + +import math +from .prior_arch import PixelNorm, EqualLinear +import torchvision +from torchvision.utils import save_image + +def GroupNorm(in_channels): + return torch.nn.GroupNorm(num_groups=in_channels//16, num_channels=in_channels, eps=1e-6, affine=False) + +Norm = GroupNorm + +class BasicBlock(nn.Module): + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv1x1(inplanes, planes) + self.norm1 = Norm(planes) + self.relu1 = nn.LeakyReLU(0.2) + self.conv2 = conv3x3(planes, planes, stride) + self.relu2 = nn.LeakyReLU(0.2) + self.norm2 = Norm(planes) + self.downsample = downsample + self.stride = stride + self.relu3 = nn.LeakyReLU(0.2) + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.norm1(out) + out = self.relu1(out) + out = self.conv2(out) + out = self.norm2(out) + out = self.relu2(out) + if self.downsample is not None: + residual = self.downsample(x) + out = out + residual + out = self.relu3(out) + return out + + +class PSPEncoder(nn.Module): + def __init__(self, block=BasicBlock, layers=[3, 4, 6, 6, 3], strides=[(2,2),(1,2),(2,2),(1,2),(2,2)]): + self.inplanes = 32 + super(PSPEncoder, self).__init__() + self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, + bias=False) + self.relu = nn.LeakyReLU(0.2) + + feature_out_dim = 256 + self.layer1 = self._make_layer(block, 32, layers[0], stride=strides[0]) + self.layer2 = self._make_layer(block, 64, layers[1], stride=strides[1]) + self.layer3 = self._make_layer(block, 128, layers[2], stride=strides[2]) + self.layer4 = self._make_layer(block, 256, layers[3], stride=strides[3]) + self.layer5 = self._make_layer(block, 512, layers[4], stride=strides[4]) + + self.layer512_to_outdim = nn.Sequential( + nn.Conv2d(512, feature_out_dim, kernel_size=1, stride=1, bias=False), + nn.LeakyReLU(0.2) + ) + self.layer256_to_512 = nn.Sequential( + nn.Conv2d(256, 512, kernel_size=1, stride=1, bias=False), + nn.LeakyReLU(0.2) + ) + + + self.down_h = 1 + for stride in strides: + self.down_h *= stride[0] + self.size_h = 32 // self.down_h * 2 + + + self.feature2w = nn.Sequential( + PixelNorm(), + EqualLinear(self.size_h*self.size_h*feature_out_dim, 512, bias=True, bias_init_val=0, lr_mul=1, activation='fused_lrelu'), + EqualLinear(512, 512, bias=True, bias_init_val=0, lr_mul=1, activation='fused_lrelu'), + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes, + kernel_size=1, stride=stride, bias=False), + nn.LeakyReLU(0.2) + ) + # GroupNorm(planes), + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + return nn.Sequential(*layers) + + + def _check_outliers(self, crop_feature, target_width): + B, C, H, W = crop_feature.size() + if W != target_width: + return F.interpolate(crop_feature, size=(H, target_width), mode='bilinear', align_corners=True) + else: + return crop_feature + + def _check_outliers_pad(self, crop_feature, start, end, max_lr_width, center_loc, extend_W): + _, _, H, W = crop_feature.size() + fill_value = crop_feature.mean().item() + if start == 0 and end == max_lr_width: + crop_feature = torchvision.transforms.Pad([extend_W//2-center_loc, 0, extend_W-W-(extend_W//2-center_loc), 0], fill=fill_value, padding_mode='constant')(crop_feature) + else: + if start == 0: + crop_feature = torchvision.transforms.Pad([extend_W-W, 0, 0, 0], fill=fill_value, padding_mode='constant')(crop_feature) + if end == max_lr_width: + crop_feature = torchvision.transforms.Pad([0, 0, extend_W-W, 0], fill=fill_value, padding_mode='constant')(crop_feature) + + # if crop_feature.size(3) != extend_W: + # print([222, crop_feature.size(), extend_W]) + # crop_feature = torchvision.transforms.Pad([(extend_W-W)//2, 0, extend_W-W-(extend_W-W)//2, 0], fill=0, padding_mode='constant')(crop_feature) + + return crop_feature + + + def forward(self, x, locs): + w_b = [] + extend_W = 32*4 + max_lr_width = x.size(3) + for b in range(locs.size(0)): #locs: 0~2048 + x_for_w = [] + for c in range(locs.size(1)): + center_loc = (locs[b][c]/4).int() + start_x = max(0, center_loc - extend_W//2) + end_x = min(center_loc + extend_W//2, max_lr_width) + crop_x = x[b:b+1, :, :, start_x:end_x].detach() + crop_x = self._check_outliers_pad(crop_x, start_x, end_x, max_lr_width, center_loc, extend_W) # + + x_for_w.append(crop_x) + # crop_x[...,62:66] = 1 + # save_image((crop_x+1)/2, 'trs_{}.png'.format(c)) + + x_for_w = torch.cat(x_for_w, dim=0) + + x_c1 = self.conv1(x_for_w) #1 + x_c1 = self.relu(x_c1) + x_l1 = self.layer1(x_c1) #2 + x_l2 = self.layer2(x_l1) #1 [2, 64, 16, 256]) + x_l3 = self.layer3(x_l2) #2 torch.Size([2, 128, 8, 128] + x_l4 = self.layer4(x_l3) #1 torch.Size([2, 256, 8, 128]) + x_l5 = self.layer5(x_l4) #2, torch.Size([2, 512, 4, 64]) + pyramid_x1 = _upsample_add(x_l5, self.layer256_to_512(x_l4)) + pyramid_x = self.layer512_to_outdim(pyramid_x1) + w_each_b = self.feature2w(pyramid_x.view(pyramid_x.size(0), -1)) # + + w_c = w_each_b + w_b.append(w_c) + w_b = torch.stack(w_b, dim=0) + + return w_b + + + + # w_b = [] + # for b in range(locs.size(0)): #locs: 0~2048 + # w_c = [] + # for c in range(locs.size(1)): + # if locs[b][c] < 2048: + # center_loc = (locs[b][c]/4).int() # 32*512 + # start_x = center_loc - 16 + # end_x = center_loc + 16 + + # crop_x0 = x[b:b+1, :, :, start_x:end_x].clone() + # crop_x = self._check_outliers_pad(crop_x0, start_x, end_x) # 1, 512, 4, 4 or 1, 512, 8, 8 + + # # save_image(crop_x[0], 'ss_{}.png'.format(c)) + # x_c1 = self.conv1(crop_x) #1 + # x_c1 = self.relu(x_c1) + # x_l1 = self.layer1(x_c1) #2 + # x_l2 = self.layer2(x_l1) #1 [2, 64, 16, 256]) + # x_l3 = self.layer3(x_l2) #2 torch.Size([2, 128, 8, 128] + # x_l4 = self.layer4(x_l3) #1 torch.Size([2, 256, 8, 128]) + # x_l5 = self.layer5(x_l4) #2, torch.Size([2, 512, 4, 64]) + # pyramid_x1 = _upsample_add(x_l5, self.layer256_to_512(x_l4)) + # pyramid_x = self.layer512_to_outdim(pyramid_x1) + + # w = self.feature2w(pyramid_x.view(1, -1)) # 1*512 + # w_c.append(w.squeeze(0)) + # else: + # w_c.append(w.squeeze(0).detach()*0) + # w_c = torch.stack(w_c, dim=0) + # w_b.append(w_c) + # w_b = torch.stack(w_b, dim=0) + # print(w_b.size()) + # return w_b #, lr + + + + + # # lr = x.clone() + # x_c1 = self.conv1(x) #1 + # x_c1 = self.relu(x_c1) + # x_l1 = self.layer1(x_c1) #2 + # x_l2 = self.layer2(x_l1) #1 [2, 64, 16, 256]) + # x_l3 = self.layer3(x_l2) #2 torch.Size([2, 128, 8, 128] + # x_l4 = self.layer4(x_l3) #1 torch.Size([2, 256, 8, 128]) + # x_l5 = self.layer5(x_l4) #2, torch.Size([2, 512, 4, 64]) B, 512, 4, 64, 17M parameters + + # pyramid_x1 = _upsample_add(x_l5, self.layer256_to_512(x_l4)) + # pyramid_x = self.layer512_to_outdim(pyramid_x1) + # # pyramid_x2 = _upsample_add(self.layer128_to_outdim(x_l3), pyramid_x1) + # B, C, H, W = pyramid_x.size() + # w_b = [] + # for b in range(locs.size(0)): #locs: 0~2048 + # w_c = [] + # for c in range(locs.size(1)): + # if locs[b][c] < 2048: + # center_loc = (locs[b][c]/4/self.down_h).int() # from 32*512 to 4*64 + # start_x = max(0, center_loc-self.size_h//2) + # end_x = min(center_loc+self.size_h//2, 512//self.down_h) + # # crop_feature = pyramid_x2[b:b+1, :, :, start_x:end_x].clone() + + # # if end_x - start_x != self.size_h: + # # bgfill = torch.zeros((B, C, H, self.size_h), dtype=pyramid_x2.dtype, layout=pyramid_x2.layout, device=pyramid_x2.device) + # # bgfill[:, :, :, self.size_h//2 - (center_loc - start_x):self.size_h//2 - (center_loc - start_x) + end_x - start_x] += pyramid_x2[b:b+1, :, :, start_x:end_x].clone() + # # crop_feature = bgfill.clone() + # # else: + # # crop_feature = pyramid_x2[b:b+1, :, :, start_x:end_x].clone() + + # crop_feature = pyramid_x[b:b+1, :, :, start_x:end_x].clone() + # crop_feature = self._check_outliers(crop_feature, self.size_h) # 1, 512, 4, 4 or 1, 512, 8, 8 + + # # crop_feature = self._check_outliers(crop_feature, self.size_h, start_x, end_x) # 1, 512, 4, 4 or 1, 512, 8, 8 + # print(crop_feature.size()) + # w = self.feature2w(crop_feature.view(1, -1)) # 1*512 + # w_c.append(w.squeeze(0)) + + # else: + # w_c.append(w.squeeze(0).detach()*0) + + # # lr[b:b+1, :, :, center_loc-1:center_loc+1] = 255 + + # w_c = torch.stack(w_c, dim=0) + # w_b.append(w_c) + # w_b = torch.stack(w_b, dim=0) + + # return w_b #, x #, lr + + + + +def GroupNorm(in_channels): + return torch.nn.GroupNorm(num_groups=in_channels//32, num_channels=in_channels, eps=1e-6, affine=False) + + +def conv1x1(in_planes, out_planes, stride=1): + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + + + +def _upsample_add(x, y): + '''Upsample and add two feature maps. + Args: + x: (Variable) top feature map to be upsampled. + y: (Variable) lateral feature map. + Returns: + (Variable) added feature map. + Note in PyTorch, when input size is odd, the upsampled feature map + with `F.upsample(..., scale_factor=2, mode='nearest')` + maybe not equal to the lateral feature map size. + e.g. + original input size: [N,_,15,15] -> + conv2d feature map size: [N,_,8,8] -> + upsampled feature map size: [N,_,16,16] + So we choose bilinear upsample which supports arbitrary output sizes. + ''' + _, _, H, W = y.size() + return F.interpolate(x, size=(H, W), mode='bilinear', align_corners=True) + y + + diff --git a/networks/rrdbnet2_arch.py b/networks/rrdbnet2_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..6714f543403ba3c1dcbd7ba1fb953b3514fe3a3d --- /dev/null +++ b/networks/rrdbnet2_arch.py @@ -0,0 +1,80 @@ +''' +from BSRGAN: https://github.com/cszn/BSRGAN +author: Kai Zhang +''' + +import functools +import torch +import torch.nn as nn +import torch.nn.functional as F +import networks.module_util as mutil + + +class ResidualDenseBlock_5C(nn.Module): + def __init__(self, nf=64, gc=32, bias=True): + super(ResidualDenseBlock_5C, self).__init__() + # gc: growth channel, i.e. intermediate channels + self.conv1 = nn.Conv2d(nf, gc, 3, 1, 1, bias=bias) + self.conv2 = nn.Conv2d(nf + gc, gc, 3, 1, 1, bias=bias) + self.conv3 = nn.Conv2d(nf + 2 * gc, gc, 3, 1, 1, bias=bias) + self.conv4 = nn.Conv2d(nf + 3 * gc, gc, 3, 1, 1, bias=bias) + self.conv5 = nn.Conv2d(nf + 4 * gc, nf, 3, 1, 1, bias=bias) + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + # initialization + mutil.initialize_weights([self.conv1, self.conv2, self.conv3, self.conv4, self.conv5], 0.1) + + def forward(self, x): + x1 = self.lrelu(self.conv1(x)) + x2 = self.lrelu(self.conv2(torch.cat((x, x1), 1))) + x3 = self.lrelu(self.conv3(torch.cat((x, x1, x2), 1))) + x4 = self.lrelu(self.conv4(torch.cat((x, x1, x2, x3), 1))) + x5 = self.conv5(torch.cat((x, x1, x2, x3, x4), 1)) + return x5 * 0.2 + x + + +class RRDB(nn.Module): + '''Residual in Residual Dense Block''' + + def __init__(self, nf, gc=32, bias=True): + super(RRDB, self).__init__() + self.RDB1 = ResidualDenseBlock_5C(nf, gc, bias) + self.RDB2 = ResidualDenseBlock_5C(nf, gc, bias) + self.RDB3 = ResidualDenseBlock_5C(nf, gc, bias) + + def forward(self, x): + out = self.RDB1(x) + out = self.RDB2(out) + out = self.RDB3(out) + return out * 0.2 + x + + +class RRDBNet(nn.Module): + def __init__(self, in_nc, out_nc, nf, nb, gc=32, sf=4, bias=True): + super(RRDBNet, self).__init__() + RRDB_block_f = functools.partial(RRDB, nf=nf, gc=gc, bias=bias) + self.sf=sf + + self.conv_first = nn.Conv2d(in_nc, nf, 3, 1, 1, bias=bias) + self.RRDB_trunk = mutil.make_layer(RRDB_block_f, nb) + self.trunk_conv = nn.Conv2d(nf, nf, 3, 1, 1, bias=bias) + #### upsampling + + self.upconv1 = nn.Conv2d(nf, nf, 3, 1, 1, bias=bias) + if self.sf==4: + self.upconv2 = nn.Conv2d(nf, nf, 3, 1, 1, bias=bias) + self.HRconv = nn.Conv2d(nf, nf, 3, 1, 1, bias=bias) + self.conv_last = nn.Conv2d(nf, out_nc, 3, 1, 1, bias=bias) + + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def forward(self, x): + fea = self.conv_first(x) + trunk = self.trunk_conv(self.RRDB_trunk(fea)) + fea = fea + trunk + fea = self.lrelu(self.upconv1(F.interpolate(fea, scale_factor=2, mode='nearest'))) + if self.sf==4: + fea = self.lrelu(self.upconv2(F.interpolate(fea, scale_factor=2, mode='nearest'))) + out = self.conv_last(self.lrelu(self.HRconv(fea))) + + return out diff --git a/networks/sr_arch.py b/networks/sr_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..6c3b2dd3c324ec4e7c0008ef477b248970d28d84 --- /dev/null +++ b/networks/sr_arch.py @@ -0,0 +1,236 @@ +import torch +from torch import nn +from torch.nn import functional as F +import torch.nn.utils.spectral_norm as SpectralNorm +import random +from .helper_arch import ResTextBlockV2, adaptive_instance_normalization + +class SRNet(nn.Module): + def __init__(self, in_channel=3, dim_channel=256): + super().__init__() + self.conv_first_32 = nn.Sequential( + SpectralNorm(nn.Conv2d(in_channel, dim_channel//4, 3, 1, 1)), + nn.LeakyReLU(0.2), + ) + self.conv_first_16 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel//4, dim_channel//2, 3, 2, 1)), + nn.LeakyReLU(0.2), + ) + self.conv_first_8 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel//2, dim_channel, 3, 2, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_body_16 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel+dim_channel//2, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_body_32 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel+dim_channel//4, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_up = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), #64*64*256 + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + ResTextBlockV2(dim_channel, dim_channel), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_final = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel//2, 3, 1, 1)), + nn.LeakyReLU(0.2), + nn.Upsample(scale_factor=2, mode='bilinear'), #128*128*256 + SpectralNorm(nn.Conv2d(dim_channel//2, dim_channel//4, 3, 1, 1)), + nn.LeakyReLU(0.2), + ResTextBlockV2(dim_channel//4, dim_channel//4), + SpectralNorm(nn.Conv2d(dim_channel//4, 3, 3, 1, 1)), + nn.Tanh() + ) + + # self.conv_priorout = nn.Sequential( + # SpectralNorm(nn.Conv2d(dim_channel, dim_channel//2, 3, 1, 1)), + # nn.LeakyReLU(0.2), + # nn.Upsample(scale_factor=2, mode='bilinear'), #128*128*256 + # SpectralNorm(nn.Conv2d(dim_channel//2, dim_channel//4, 3, 1, 1)), + # nn.LeakyReLU(0.2), + # ResTextBlockV2(dim_channel//4, dim_channel//4), + # SpectralNorm(nn.Conv2d(dim_channel//4, 3, 3, 1, 1)), + # nn.Tanh() + # ) + + self.conv_32_scale = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_32_shift = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_32_fuse = nn.Sequential( + ResTextBlockV2(2*dim_channel, dim_channel) + ) + + self.conv_32_to256 = nn.Sequential( + SpectralNorm(nn.Conv2d(512, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_64_scale = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_64_shift = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_64_fuse = nn.Sequential( + ResTextBlockV2(2*dim_channel, dim_channel) + ) + + def forward(self, lq, priors64, priors32, locs): # + # lq_features:b*512*8*512 + # priors: 8, 16,32,64,128 + # locs: b*32, center+width for 128*2048 0~1 + # locs: b*16, center for 128*2048, 0~2048 + + + single_sr = True + + + lq_f_32 = self.conv_first_32(lq) + lq_f_16 = self.conv_first_16(lq_f_32) + lq_f_8 = self.conv_first_8(lq_f_16) + + sq_f_16 = self.conv_body_16(torch.cat([F.interpolate(lq_f_8, scale_factor=2, mode='bilinear'), lq_f_16], dim=1)) + sq_f_32 = self.conv_body_32(torch.cat([F.interpolate(sq_f_16, scale_factor=2, mode='bilinear'), lq_f_32], dim=1)) # 256*32*32 + + + if priors32 is not None: + sq_f_32_ori = sq_f_32.clone() + # sq_f_32_res = sq_f_32.clone().detach()*0 + prior_32_align = torch.zeros_like(sq_f_32_ori) + prior_32_mask = torch.zeros((sq_f_32_ori.size(0), 1, sq_f_32_ori.size(2), sq_f_32_ori.size(3)), dtype=sq_f_32_ori.dtype, layout=sq_f_32_ori.layout, device=sq_f_32_ori.device) + for b, p_32 in enumerate(priors32): #512*32*32, different batch + p_32_256 = self.conv_32_to256(p_32.clone()) + for c in range(p_32_256.size(0)): # + center = (locs[b][c].detach()/4.0).int() # + width = 16 + + if center < width: + x1 = 0 #lq feature left + y1 = max(16 - center, 0) + else: + x1 = center - width + y1 = max(16 - width, 0) + # y1 = 16 - width + if center + width > sq_f_32.size(-1): + x2 = sq_f_32.size(-1) #lq feature right + else: + x2 = center + width + y2 = y1 + (x2 - x1) + + ''' + center align + ''' + # y1 = 16 - torch.div(x2-x1, 2, rounding_mode='trunc') + y2 = y1 + x2 - x1 + + if single_sr: + char_prior_f = p_32_256[c:c+1, :, :, y1:y2].clone() #prior + char_lq_f = sq_f_32[b:b+1, :, :, x1:x2].clone() + adain_prior_f = adaptive_instance_normalization(char_prior_f, char_lq_f) + fuse_32_prior = self.conv_32_fuse(torch.cat((adain_prior_f, char_lq_f), dim=1)) + scale = self.conv_32_scale(fuse_32_prior) + shift = self.conv_32_shift(fuse_32_prior) + prior_32_align[b, :, :, x1:x2] = prior_32_align[b, :, :, x1:x2] + sq_f_32[b, :, :, x1:x2].clone() * scale[0,...] + shift[0,...] + else: + prior_32_align[b, :, :, x1:x2] = prior_32_align[b, :, :, x1:x2] + p_32_256[c:c+1, :, :, y1:y2].clone() + # prior_32_mask[b, :, :, x1:x2] += 1.0 + + # prior_32_mask[prior_32_mask<2]=1.0 + # prior_32_align = prior_32_align / prior_32_mask.repeat(1, prior_32_align.size(1), 1, 1) + + if single_sr: + sq_pf_32_out = sq_f_32_ori + prior_32_align + else: + sq_f_32_norm = adaptive_instance_normalization(prior_32_align, sq_f_32) + sq_f_32_fuse = self.conv_32_fuse(torch.cat((sq_f_32_norm, sq_f_32), dim=1)) + scale_32 = self.conv_32_scale(sq_f_32_fuse) + shift_32 = self.conv_32_shift(sq_f_32_fuse) + sq_f_32_res = sq_f_32_ori * scale_32 + shift_32 + sq_pf_32_out = sq_f_32_ori + sq_f_32_res + + else: + sq_pf_32_out = sq_f_32.clone() + + + sq_f_64 = self.conv_up(sq_pf_32_out) #64*1024 + + sq_f_64_ori = sq_f_64.clone() + prior_64_align = torch.zeros_like(sq_f_64_ori) + prior_64_mask = torch.zeros((sq_f_64_ori.size(0), 1, sq_f_64_ori.size(2), sq_f_64_ori.size(3)), dtype=sq_f_64_ori.dtype, layout=sq_f_64_ori.layout, device=sq_f_64_ori.device) + + + for b, p_64_prior in enumerate(priors64): #512*8*8, 512*16*16, 512*32*32, 256*64*64, 128*128*128 different batch + p_64 = p_64_prior.clone() #.detach() #no backward to prior + for c in range(p_64.size(0)): # for each character + center = (locs[b][c].detach()/2.0).int() #+ random.randint(-4,4)### no backward + width = 32 + if center < width: + x1 = 0 + y1 = max(32 - center, 0) + else: + x1 = center -width + y1 = max(32 - width, 0) + if center + width > sq_f_64.size(-1): + x2 = sq_f_64.size(-1) + else: + x2 = center + width + + ''' + center align + ''' + # y1 = 32 - torch.div(x2-x1, 2, rounding_mode='trunc') + y2 = y1 + x2 - x1 + + if single_sr: + char_prior_f = p_64[c:c+1, :, :, y1:y2].clone() + char_lq_f = sq_f_64[b:b+1, :, :, x1:x2].clone() + adain_prior_f = adaptive_instance_normalization(char_prior_f, char_lq_f) + + fuse_64_prior = self.conv_64_fuse(torch.cat((adain_prior_f, char_lq_f), dim=1)) + scale = self.conv_64_scale(fuse_64_prior) + shift = self.conv_64_shift(fuse_64_prior) + prior_64_align[b, :, :, x1:x2] = prior_64_align[b, :, :, x1:x2] + sq_f_64[b, :, :, x1:x2].clone() * scale[0,...] + shift[0,...] + else: + prior_64_align[b, :, :, x1:x2] = prior_64_align[b, :, :, x1:x2] + p_64[c:c+1, :, :, y1:y2].clone() + # prior_64_mask[b, :, :, x1:x2] += 1.0 + + + # prior_64_mask[prior_64_mask<2]=1.0 + # prior_64_align = prior_64_align / prior_64_mask.repeat(1, prior_64_align.size(1), 1, 1) + if single_sr: + sq_pf_64 = sq_f_64_ori + prior_64_align + else: + sq_f_64_norm = adaptive_instance_normalization(prior_64_align, sq_f_64_ori) + sq_f_64_fuse = self.conv_64_fuse(torch.cat((sq_f_64_norm, sq_f_64_ori), dim=1)) + scale_64 = self.conv_64_scale(sq_f_64_fuse) + shift_64 = self.conv_64_shift(sq_f_64_fuse) + sq_f_64_res = sq_f_64_ori * scale_64 + shift_64 + sq_pf_64 = sq_f_64_ori + sq_f_64_res + + f256 = self.conv_final(sq_pf_64) + + # adain_lr2prior = adaptive_instance_normalization(prior_full_64, F.interpolate(sq_f_32_ori, scale_factor=2, mode='bilinear')) + # prior_out = self.conv_priorout(adain_lr2prior) + + return f256 #prior_out \ No newline at end of file diff --git a/networks/sr_arch_singlec.py b/networks/sr_arch_singlec.py new file mode 100644 index 0000000000000000000000000000000000000000..dd5f901d2dce3f315c10be93a66c0298f470d1ca --- /dev/null +++ b/networks/sr_arch_singlec.py @@ -0,0 +1,193 @@ +import torch +from torch import nn +from torch.nn import functional as F +import torch.nn.utils.spectral_norm as SpectralNorm +import random +from .helper_arch import ResTextBlockV2, adaptive_instance_normalization + +class SRNet(nn.Module): + def __init__(self, in_channel=3, dim_channel=256): + super().__init__() + self.conv_first_32 = nn.Sequential( + SpectralNorm(nn.Conv2d(in_channel, dim_channel//4, 3, 1, 1)), + nn.LeakyReLU(0.2), + ) + self.conv_first_16 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel//4, dim_channel//2, 3, 2, 1)), + nn.LeakyReLU(0.2), + ) + self.conv_first_8 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel//2, dim_channel, 3, 2, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_body_16 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel+dim_channel//2, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_body_32 = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel+dim_channel//4, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_up = nn.Sequential( + nn.Upsample(scale_factor=2, mode='bilinear'), #64*64*256 + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + ResTextBlockV2(dim_channel, dim_channel), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_final = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel//2, 3, 1, 1)), + nn.LeakyReLU(0.2), + nn.Upsample(scale_factor=2, mode='bilinear'), #128*128*256 + SpectralNorm(nn.Conv2d(dim_channel//2, dim_channel//4, 3, 1, 1)), + nn.LeakyReLU(0.2), + ResTextBlockV2(dim_channel//4, dim_channel//4), + SpectralNorm(nn.Conv2d(dim_channel//4, 3, 3, 1, 1)), + nn.Tanh() + ) + + + self.conv_32_scale = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_32_shift = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_32_fuse = nn.Sequential( + ResTextBlockV2(2*dim_channel, dim_channel) + ) + + self.conv_32_to256 = nn.Sequential( + SpectralNorm(nn.Conv2d(512, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + + self.conv_64_scale = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_64_shift = nn.Sequential( + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + nn.LeakyReLU(0.2), + SpectralNorm(nn.Conv2d(dim_channel, dim_channel, 3, 1, 1)), + ) + self.conv_64_fuse = nn.Sequential( + ResTextBlockV2(2*dim_channel, dim_channel) + ) + + def forward(self, lq, priors64, priors32, locs): # + + lq_f_32 = self.conv_first_32(lq) + lq_f_16 = self.conv_first_16(lq_f_32) + lq_f_8 = self.conv_first_8(lq_f_16) + + sq_f_16 = self.conv_body_16(torch.cat([F.interpolate(lq_f_8, scale_factor=2, mode='bilinear'), lq_f_16], dim=1)) + sq_f_32 = self.conv_body_32(torch.cat([F.interpolate(sq_f_16, scale_factor=2, mode='bilinear'), lq_f_32], dim=1)) # + + + if priors32 is not None: + sq_f_32_ori = sq_f_32.clone() + sq_f_32_res = sq_f_32.clone().detach()*0 + for b, p_32 in enumerate(priors32): # + p_32_256 = self.conv_32_to256(p_32.clone().detach()) + for c in range(p_32_256.size(0)): # + center = int(locs[b][c].item()/4.0) #+ random.randint(-2,2)### no backward + width = 16 + + if center < width: + x1 = 0 #lq feature left + y1 = max(16 - center, 0) + else: + x1 = center - width + y1 = max(16 - width, 0) + # y1 = 16 - width + if center + width > sq_f_32.size(-1): + x2 = sq_f_32.size(-1) #lq feature right + else: + x2 = center + width + y2 = y1 + (x2 - x1) + + ''' + center align + ''' + y1 = 16 - torch.div(x2-x1, 2, rounding_mode='trunc') + y2 = y1 + x2 - x1 + + char_prior_f = p_32_256[c:c+1, :, :, y1:y2].clone() #prior + char_lq_f = sq_f_32[b:b+1, :, :, x1:x2].clone() + adain_prior_f = adaptive_instance_normalization(char_prior_f, char_lq_f) + fuse_32_prior = self.conv_32_fuse(torch.cat((adain_prior_f, char_lq_f), dim=1)) + scale = self.conv_32_scale(fuse_32_prior) + shift = self.conv_32_shift(fuse_32_prior) + + sq_f_32_res[b, :, :, x1:x2] = sq_f_32_res[b, :, :, x1:x2] + sq_f_32[b, :, :, x1:x2].clone() * scale[0,...] + shift[0,...] + + sq_pf_32_out = sq_f_32_ori + sq_f_32_res + + else: + sq_pf_32_out = sq_f_32.clone() + + + sq_f_64 = self.conv_up(sq_pf_32_out) #64*1024 + + + sq_f_64_ori = sq_f_64.clone() + sq_f_64_res = sq_f_64.clone().detach() * 0 + + # prior_full_64 = sq_f_64.clone().detach() * 0 + if priors64 is not None: + for b, p_64_prior in enumerate(priors64): # + p_64 = p_64_prior.clone().detach() #no backward to prior + for c in range(p_64.size(0)): # for each character + center = int(locs[b][c].item()/2.0) #+ random.randint(-4,4)### no backward + width = 32 + + if center < width: + x1 = 0 + y1 = max(32 - center, 0) + else: + x1 = center -width + y1 = max(32 - width, 0) + if center + width > sq_f_64.size(-1): + x2 = sq_f_64.size(-1) + else: + x2 = center + width + + ''' + center align + ''' + y1 = 32 - torch.div(x2-x1, 2, rounding_mode='trunc') + y2 = y1 + x2 - x1 + + char_prior_f = p_64[c:c+1, :, :, y1:y2].clone() + char_lq_f = sq_f_64[b:b+1, :, :, x1:x2].clone() + adain_prior_f = adaptive_instance_normalization(char_prior_f, char_lq_f) + + fuse_64_prior = self.conv_64_fuse(torch.cat((adain_prior_f, char_lq_f), dim=1)) + scale = self.conv_64_scale(fuse_64_prior) + shift = self.conv_64_shift(fuse_64_prior) + sq_f_64_res[b, :, :, x1:x2] = sq_f_64_res[b, :, :, x1:x2] + sq_f_64[b, :, :, x1:x2].clone() * scale[0,...] + shift[0,...] + + # prior_full_64[b, :, :, x1:x2] = prior_full_64[b, :, :, x1:x2] + char_prior_f.clone() + sq_pf_64 = sq_f_64_ori + sq_f_64_res + else: + sq_pf_64 = sq_f_64_ori.clone() + + f256 = self.conv_final(sq_pf_64) + + + # adain_lr2prior = adaptive_instance_normalization(prior_full_64, F.interpolate(sq_f_32_ori, scale_factor=2, mode='bilinear')) + # prior_out = self.conv_priorout(adain_lr2prior) + + return f256 #prior_out \ No newline at end of file diff --git a/networks/transocr_arch.py b/networks/transocr_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..57f15089b41de95d0649da9264e3f1c373e0f9a7 --- /dev/null +++ b/networks/transocr_arch.py @@ -0,0 +1,365 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math, copy +import numpy as np +from torch.autograd import Variable + + + +class BasicBlock(nn.Module): + + def __init__(self, inplanes, planes, downsample): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=1, padding=1) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample != None: + residual = self.downsample(residual) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, num_in, block, layers): + super(ResNet, self).__init__() + + self.conv1 = nn.Conv2d(num_in, 64, kernel_size=3, stride=1, padding=1) + self.bn1 = nn.BatchNorm2d(64) + self.relu1 = nn.ReLU(inplace=True) + self.pool = nn.MaxPool2d((2, 2), (2, 2)) + + self.conv2 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1) + self.bn2 = nn.BatchNorm2d(128) + self.relu2 = nn.ReLU(inplace=True) + + self.layer1_pool = nn.MaxPool2d((2, 2), (2, 2)) + self.layer1 = self._make_layer(block, 128, 256, layers[0]) + self.layer1_conv = nn.Conv2d(256, 256, 3, 1, 1) + self.layer1_bn = nn.BatchNorm2d(256) + self.layer1_relu = nn.ReLU(inplace=True) + + self.layer2_pool = nn.MaxPool2d((2, 2), (2, 2)) + self.layer2 = self._make_layer(block, 256, 256, layers[1]) + self.layer2_conv = nn.Conv2d(256, 256, 3, 1, 1) + self.layer2_bn = nn.BatchNorm2d(256) + self.layer2_relu = nn.ReLU(inplace=True) + + self.layer3_pool = nn.MaxPool2d((2, 2), (2, 2)) + self.layer3 = self._make_layer(block, 256, 512, layers[2]) + self.layer3_conv = nn.Conv2d(512, 512, 3, 1, 1) + self.layer3_bn = nn.BatchNorm2d(512) + self.layer3_relu = nn.ReLU(inplace=True) + + self.layer4_pool = nn.MaxPool2d((2, 2), (2, 2)) + self.layer4 = self._make_layer(block, 512, 512, layers[3]) + self.layer4_conv2 = nn.Conv2d(512, 1024, 3, 1, 1) + self.layer4_conv2_bn = nn.BatchNorm2d(1024) + self.layer4_conv2_relu = nn.ReLU(inplace=True) + + def _make_layer(self, block, inplanes, planes, blocks): + + if inplanes != planes: + downsample = nn.Sequential( + nn.Conv2d(inplanes, planes, 3, 1, 1), + nn.BatchNorm2d(planes), ) + else: + downsample = None + layers = [] + layers.append(block(inplanes, planes, downsample)) + for i in range(1, blocks): + layers.append(block(planes, planes, downsample=None)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu1(x) + x = self.pool(x) + + x = self.conv2(x) + x = self.bn2(x) + x = self.relu2(x) + + x = self.layer1_pool(x) + x = self.layer1(x) + x = self.layer1_conv(x) + x = self.layer1_bn(x) + x = self.layer1_relu(x) + + + x = self.layer2_pool(x) + x = self.layer2(x) + x = self.layer2_conv(x) + x = self.layer2_bn(x) + x = self.layer2_relu(x) + + x = self.layer3_pool(x) + x = self.layer3(x) + x = self.layer3_conv(x) + x = self.layer3_bn(x) + x = self.layer3_relu(x) + + x = self.layer4(x) + x = self.layer4_conv2(x) + x = self.layer4_conv2_bn(x) + x = self.layer4_conv2_relu(x) + + return x + + +def clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for _ in range(N)]) + + +class PositionalEncoding(nn.Module): + + def __init__(self, d_model, dropout, max_len=7000): + super(PositionalEncoding, self).__init__() + self.dropout = nn.Dropout(p=dropout) + + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len).unsqueeze(1).float() + div_term = torch.exp(torch.arange(0, d_model, 2).float() * + -(math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) + + def forward(self, x): + x = x + Variable(self.pe[:, :x.size(1)], + requires_grad=False) + return self.dropout(x) + + +class MultiHeadedAttention(nn.Module): + def __init__(self, h, d_model, dropout=0.1, compress_attention=False): + super(MultiHeadedAttention, self).__init__() + assert d_model % h == 0 + self.d_k = d_model // h + self.h = h + self.linears = clones(nn.Linear(d_model, d_model), 4) + self.attn = None + self.dropout = nn.Dropout(p=dropout) + self.compress_attention = compress_attention + self.compress_attention_linear = nn.Linear(h, 1) + + def forward(self, query, key, value, mask=None, align=None): + if mask is not None: + mask = mask.unsqueeze(1) + nbatches = query.size(0) + + query, key, value = \ + [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2) + for l, x in zip(self.linears, (query, key, value))] + + x, attention_map = attention(query, key, value, mask=mask, + dropout=self.dropout, align=align) + + x = x.transpose(1, 2).contiguous() \ + .view(nbatches, -1, self.h * self.d_k) + + if self.compress_attention: + batch, head, s1, s2 = attention_map.shape + attention_map = attention_map.permute(0, 2, 3, 1).contiguous() + attention_map = self.compress_attention_linear(attention_map).permute(0, 3, 1, 2).contiguous() + return self.linears[-1](x), attention_map + + +def subsequent_mask(size): + attn_shape = (1, size, size) + subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8') + return torch.from_numpy(subsequent_mask) == 0 + + +def attention(query, key, value, mask=None, dropout=None, align=None): + + d_k = query.size(-1) + + scores = torch.matmul(query, key.transpose(-2, -1)) \ + / math.sqrt(d_k) + if mask is not None: + scores = scores.masked_fill(mask == 0, float('-inf')) + else: + pass + p_attn = F.softmax(scores, dim=-1) + + if dropout is not None: + p_attn = dropout(p_attn) + return torch.matmul(p_attn, value), p_attn + + +class LayerNorm(nn.Module): + + def __init__(self, features, eps=1e-6): + super(LayerNorm, self).__init__() + self.a_2 = nn.Parameter(torch.ones(features)) + self.b_2 = nn.Parameter(torch.zeros(features)) + self.eps = eps + + def forward(self, x): + mean = x.mean(-1, keepdim=True) + std = x.std(-1, keepdim=True) + return self.a_2 * (x - mean) / (std + self.eps) + self.b_2 + + +class PositionwiseFeedForward(nn.Module): + + def __init__(self, d_model, d_ff, dropout=0.1): + super(PositionwiseFeedForward, self).__init__() + self.w_1 = nn.Linear(d_model, d_ff) + self.w_2 = nn.Linear(d_ff, d_model) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + return self.w_2(self.dropout(F.relu(self.w_1(x)))) + + +class Generator(nn.Module): + + def __init__(self, d_model, vocab, norm=False): + super(Generator, self).__init__() + self.proj = nn.Linear(d_model, vocab) + self.norm = norm + self.activation = nn.ReLU() #nn.Sigmoid() + + def forward(self, x): + if self.norm: + return self.activation(self.proj(x)) + else: + return self.proj(x) + + +class Embeddings(nn.Module): + def __init__(self, d_model, vocab): + super(Embeddings, self).__init__() + self.lut = nn.Embedding(vocab, d_model) + self.d_model = d_model + + def forward(self, x): + embed = self.lut(x) * math.sqrt(self.d_model) + return embed + +class TransformerBlock(nn.Module): + """Transformer Block""" + def __init__(self, dim, num_heads, ff_dim, dropout): + super(TransformerBlock, self).__init__() + self.attn = MultiHeadedAttention(h=num_heads, d_model=dim, dropout=dropout) + self.proj = nn.Linear(dim, dim) + self.norm1 = nn.LayerNorm(dim, eps=1e-6) + self.pwff = PositionwiseFeedForward(dim, ff_dim) + self.norm2 = nn.LayerNorm(dim, eps=1e-6) + self.drop = nn.Dropout(dropout) + + def forward(self, x, mask): + x = self.norm1(x) + h = self.drop(self.proj(self.attn(x, x, x, mask)[0])) + x = x + h + h = self.drop(self.pwff(self.norm2(x))) + x = x + h + return x + +class Decoder(nn.Module): + + def __init__(self): + super(Decoder, self).__init__() + + self.mask_multihead = MultiHeadedAttention(h=4, d_model=1024, dropout=0.1) + self.mul_layernorm1 = LayerNorm(features=1024) + + self.multihead = MultiHeadedAttention(h=4, d_model=1024, dropout=0.1, compress_attention=False) + self.mul_layernorm2 = LayerNorm(features=1024) + + self.pff = PositionwiseFeedForward(1024, 2048) + self.mul_layernorm3 = LayerNorm(features=1024) + + def forward(self, text, conv_feature): + text_max_length = text.shape[1] + mask = subsequent_mask(text_max_length).cuda() + + result = text + result = self.mul_layernorm1(result + self.mask_multihead(result, result, result, mask=mask)[0]) + + b, c, h, w = conv_feature.shape + conv_feature = conv_feature.view(b, c, h * w).permute(0, 2, 1).contiguous() + word_image_align, attention_map = self.multihead(result, conv_feature, conv_feature, mask=None) + result = self.mul_layernorm2(result + word_image_align) + + result = self.mul_layernorm3(result + self.pff(result)) + + return result, attention_map + + +class TransformerOCR(nn.Module): + def __init__(self, word_n_class=6738, use_bbox=False): + super(TransformerOCR, self).__init__() + self.word_n_class = word_n_class + self.use_bbox = use_bbox + self.embedding_word = Embeddings(512, self.word_n_class) + self.pe = PositionalEncoding(d_model=512, dropout=0.1, max_len=7000) + self.encoder = ResNet(num_in=3, block=BasicBlock, layers=[3,4,6,3]).cuda() + self.decoder = Decoder() + self.generator_word = Generator(1024, self.word_n_class) + if self.use_bbox: + self.generator_loc = Generator(1024, 1, True) + + + + def forward(self, image, text_length, text_input, conv_feature=None): + + if conv_feature is None: + conv_feature = self.encoder(image) + + if text_length is None: + return { + 'conv': conv_feature, + } + + text_embedding = self.embedding_word(text_input) + postion_embedding = self.pe(torch.zeros(text_embedding.shape).cuda()).cuda() + text_input_with_pe = torch.cat([text_embedding, postion_embedding], 2) + text_input_with_pe, attention_map = self.decoder(text_input_with_pe, conv_feature) + word_decoder_result = self.generator_word(text_input_with_pe) + if self.use_bbox: + word_loc_result = self.generator_loc(text_input_with_pe) + else: + word_loc_result = None + + return { + 'pred': word_decoder_result, + 'map': attention_map, + 'conv': conv_feature, + 'loc': word_loc_result + } + + + + +if __name__ == '__main__': + net = ResNet(num_in=3, block=BasicBlock, layers=[3, 4, 6, 3]).cuda() + image = torch.Tensor(8, 3, 64, 64).cuda() + result = net(image) + print(result.shape) + pass + + diff --git a/networks/w_encoder_arch.py b/networks/w_encoder_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..b1406b1be9867e42200c6b046de9d599bdfe5aeb --- /dev/null +++ b/networks/w_encoder_arch.py @@ -0,0 +1,218 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from .prior_arch import PixelNorm, EqualLinear + + +class BasicBlock(nn.Module): + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv1x1(inplanes, planes) + self.gn1 = GroupNorm(planes) + self.relu = nn.LeakyReLU(0.2, inplace=True) + self.conv2 = conv3x3(planes, planes, stride) + self.gn2 = GroupNorm(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.gn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.gn2(out) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + +class WEncoder(nn.Module): + def __init__(self, block=BasicBlock, layers=[3, 4, 6, 6, 3], strides=[2,1,2,1,2]): + self.inplanes = 32 + super(WEncoder, self).__init__() + self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, + bias=False) + self.relu = nn.LeakyReLU(0.2, inplace=True) + + feature_out_dim = 512 + self.layer1 = self._make_layer(block, 32, layers[0], stride=strides[0]) + self.layer2 = self._make_layer(block, 64, layers[1], stride=strides[1]) + self.layer3 = self._make_layer(block, 128, layers[2], stride=strides[2]) + self.layer4 = self._make_layer(block, 256, layers[3], stride=strides[3]) + self.layer5 = self._make_layer(block, feature_out_dim, layers[4], stride=strides[4]) + + + self.down_h = 1 + for stride in strides: + self.down_h *= stride + self.size_h = 32 // self.down_h + + + self.feature2w = nn.Sequential( + PixelNorm(), + EqualLinear(self.size_h*self.size_h*feature_out_dim, 512, bias=True, bias_init_val=0, lr_mul=1, + activation='fused_lrelu'), + EqualLinear(512, 512, bias=True, bias_init_val=0, lr_mul=1, + activation='fused_lrelu') + # EqualLinear(self.size_h*self.size_h*feature_out_dim, 512, bias=True), + # EqualLinear(512, 512, bias=True) + ) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes, + kernel_size=1, stride=stride, bias=False), + ) + # GroupNorm(planes), + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _check_outliers(self, crop_feature, target_width): + _, _, H, W = crop_feature.size() + if W != target_width: + return F.interpolate(crop_feature, size=(H, target_width), mode='bilinear', align_corners=True) + else: + return crop_feature + + + def forward(self, x, locs): + # lr = x.clone() + x = self.conv1(x) + x = self.relu(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.layer5(x) # B, 512, 4, 64, 17M parameters + + B, C, H, W = x.size() + + # lr = F.interpolate(lr, (x.size(2), x.size(3))) + w_b = [] + for b in range(locs.size(0)): #locs: 0~2048 + w_c = [] + for c in range(locs.size(1)): + if locs[b][c] < 2048: + center_loc = (locs[b][c]/4/self.down_h).int() # from 32*512 to 4*64 + start_x = max(0, center_loc-self.size_h//2) + end_x = min(center_loc+self.size_h//2, 512//self.down_h) + + # crop_feature = x[b:b+1, :, :, start_x:end_x].clone() + # crop_feature = self._check_outliers(crop_feature, self.size_h) # 1, 512, 4, 4 or 1, 512, 8, 8 + + if end_x - start_x != self.size_h: + bgfill = torch.zeros((B, C, H, self.size_h), dtype=x.dtype, layout=x.layout, device=x.device) + bgfill[:, :, :, self.size_h//2 - (center_loc - start_x):self.size_h//2 - (center_loc - start_x) + end_x - start_x] += x[b:b+1, :, :, start_x:end_x].clone() + crop_feature = bgfill.clone() + else: + crop_feature = x[b:b+1, :, :, start_x:end_x].clone() + w = self.feature2w(crop_feature.view(1, -1)) # 1*512 + w_c.append(w.squeeze(0)) + + else: + w_c.append(w.squeeze(0).detach()*0) + + w_c = torch.stack(w_c, dim=0) + w_b.append(w_c) + w_b = torch.stack(w_b, dim=0) + + return w_b #, lr + + + +def GroupNorm(in_channels): + return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=False) + + +def conv1x1(in_planes, out_planes, stride=1): + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + + + +def _upsample_add(x, y): + '''Upsample and add two feature maps. + Args: + x: (Variable) top feature map to be upsampled. + y: (Variable) lateral feature map. + Returns: + (Variable) added feature map. + Note in PyTorch, when input size is odd, the upsampled feature map + with `F.upsample(..., scale_factor=2, mode='nearest')` + maybe not equal to the lateral feature map size. + e.g. + original input size: [N,_,15,15] -> + conv2d feature map size: [N,_,8,8] -> + upsampled feature map size: [N,_,16,16] + So we choose bilinear upsample which supports arbitrary output sizes. + ''' + _, _, H, W = y.size() + return F.interpolate(x, size=(H, W), mode='bilinear', align_corners=True) + y + + + + +if __name__ == '__main__': + from .helper_arch import network_param + + device = 'cuda' + input = torch.randn(2, 3, 32, 512).to(device) # + + test_list = [64] + for i in range(1, 8): + test_list.append(64+128*i) + + for i in range(8, 16): + test_list.append(2048) + + + + locs = torch.Tensor(test_list).unsqueeze(0) + locs = locs.repeat(2, 1).to(device) + net = WEncoder().to(device) + ''' + strides=[2,1,2,1,1] output h is 8 + Encoder is 12.97M + F2W+Encoder is 17.04 M + + strides=[2,1,2,1,2] output h is 4 + Encoder is 12.97M + F2W is 4.46 M + + ''' + + output = net(input, locs) + print([input.size(), output.size(), locs.size(), network_param(net)]) + #[torch.Size([2, 3, 32, 512]), torch.Size([2, 16, 512]), torch.Size([2, 16]), 17.43344] + + # import numpy as np + # import cv2 + # sr_results = lr[0].permute(1, 2, 0) + # sr_results = sr_results.float().cpu().numpy() + + # cv2.imwrite('./tmp.png', sr_results) + + diff --git a/op/__init__.py b/op/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0918d92285955855be89f00096b888ee5597ce3 --- /dev/null +++ b/op/__init__.py @@ -0,0 +1,2 @@ +from .fused_act import FusedLeakyReLU, fused_leaky_relu +from .upfirdn2d import upfirdn2d diff --git a/op/conv2d_gradfix.py b/op/conv2d_gradfix.py new file mode 100644 index 0000000000000000000000000000000000000000..64229c5a7fd04292140abac1f490619963009328 --- /dev/null +++ b/op/conv2d_gradfix.py @@ -0,0 +1,227 @@ +import contextlib +import warnings + +import torch +from torch import autograd +from torch.nn import functional as F + +enabled = True +weight_gradients_disabled = False + + +@contextlib.contextmanager +def no_weight_gradients(): + global weight_gradients_disabled + + old = weight_gradients_disabled + weight_gradients_disabled = True + yield + weight_gradients_disabled = old + + +def conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + if could_use_op(input): + return conv2d_gradfix( + transpose=False, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=0, + dilation=dilation, + groups=groups, + ).apply(input, weight, bias) + + return F.conv2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + ) + + +def conv_transpose2d( + input, + weight, + bias=None, + stride=1, + padding=0, + output_padding=0, + groups=1, + dilation=1, +): + if could_use_op(input): + return conv2d_gradfix( + transpose=True, + weight_shape=weight.shape, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=groups, + dilation=dilation, + ).apply(input, weight, bias) + + return F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + stride=stride, + padding=padding, + output_padding=output_padding, + dilation=dilation, + groups=groups, + ) + + +def could_use_op(input): + if (not enabled) or (not torch.backends.cudnn.enabled): + return False + + if input.device.type != "cuda": + return False + + if any(torch.__version__.startswith(x) for x in ["1.7.", "1.8."]): + return True + + warnings.warn( + f"conv2d_gradfix not supported on PyTorch {torch.__version__}. Falling back to torch.nn.functional.conv2d()." + ) + + return False + + +def ensure_tuple(xs, ndim): + xs = tuple(xs) if isinstance(xs, (tuple, list)) else (xs,) * ndim + + return xs + + +conv2d_gradfix_cache = dict() + + +def conv2d_gradfix( + transpose, weight_shape, stride, padding, output_padding, dilation, groups +): + ndim = 2 + weight_shape = tuple(weight_shape) + stride = ensure_tuple(stride, ndim) + padding = ensure_tuple(padding, ndim) + output_padding = ensure_tuple(output_padding, ndim) + dilation = ensure_tuple(dilation, ndim) + + key = (transpose, weight_shape, stride, padding, output_padding, dilation, groups) + if key in conv2d_gradfix_cache: + return conv2d_gradfix_cache[key] + + common_kwargs = dict( + stride=stride, padding=padding, dilation=dilation, groups=groups + ) + + def calc_output_padding(input_shape, output_shape): + if transpose: + return [0, 0] + + return [ + input_shape[i + 2] + - (output_shape[i + 2] - 1) * stride[i] + - (1 - 2 * padding[i]) + - dilation[i] * (weight_shape[i + 2] - 1) + for i in range(ndim) + ] + + class Conv2d(autograd.Function): + @staticmethod + def forward(ctx, input, weight, bias): + if not transpose: + out = F.conv2d(input=input, weight=weight, bias=bias, **common_kwargs) + + else: + out = F.conv_transpose2d( + input=input, + weight=weight, + bias=bias, + output_padding=output_padding, + **common_kwargs, + ) + + ctx.save_for_backward(input, weight) + + return out + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input, grad_weight, grad_bias = None, None, None + + if ctx.needs_input_grad[0]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape + ) + grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, weight, None) + + if ctx.needs_input_grad[1] and not weight_gradients_disabled: + grad_weight = Conv2dGradWeight.apply(grad_output, input) + + if ctx.needs_input_grad[2]: + grad_bias = grad_output.sum((0, 2, 3)) + + return grad_input, grad_weight, grad_bias + + class Conv2dGradWeight(autograd.Function): + @staticmethod + def forward(ctx, grad_output, input): + op = torch._C._jit_get_operation( + "aten::cudnn_convolution_backward_weight" + if not transpose + else "aten::cudnn_convolution_transpose_backward_weight" + ) + flags = [ + torch.backends.cudnn.benchmark, + torch.backends.cudnn.deterministic, + torch.backends.cudnn.allow_tf32, + ] + grad_weight = op( + weight_shape, + grad_output, + input, + padding, + stride, + dilation, + groups, + *flags, + ) + ctx.save_for_backward(grad_output, input) + + return grad_weight + + @staticmethod + def backward(ctx, grad_grad_weight): + grad_output, input = ctx.saved_tensors + grad_grad_output, grad_grad_input = None, None + + if ctx.needs_input_grad[0]: + grad_grad_output = Conv2d.apply(input, grad_grad_weight, None) + + if ctx.needs_input_grad[1]: + p = calc_output_padding( + input_shape=input.shape, output_shape=grad_output.shape + ) + grad_grad_input = conv2d_gradfix( + transpose=(not transpose), + weight_shape=weight_shape, + output_padding=p, + **common_kwargs, + ).apply(grad_output, grad_grad_weight, None) + + return grad_grad_output, grad_grad_input + + conv2d_gradfix_cache[key] = Conv2d + + return Conv2d diff --git a/op/fused_act.py b/op/fused_act.py new file mode 100644 index 0000000000000000000000000000000000000000..5d46e10d6cb35867dbc4871e1c576141f4fe2efe --- /dev/null +++ b/op/fused_act.py @@ -0,0 +1,127 @@ +import os + +import torch +from torch import nn +from torch.nn import functional as F +from torch.autograd import Function +from torch.utils.cpp_extension import load + + +module_path = os.path.dirname(__file__) +fused = load( + "fused", + sources=[ + os.path.join(module_path, "fused_bias_act.cpp"), + os.path.join(module_path, "fused_bias_act_kernel.cu"), + ], +) + + +class FusedLeakyReLUFunctionBackward(Function): + @staticmethod + def forward(ctx, grad_output, out, bias, negative_slope, scale): + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + empty = grad_output.new_empty(0) + + grad_input = fused.fused_bias_act( + grad_output.contiguous(), empty, out, 3, 1, negative_slope, scale + ) + + dim = [0] + + if grad_input.ndim > 2: + dim += list(range(2, grad_input.ndim)) + + if bias: + grad_bias = grad_input.sum(dim).detach() + + else: + grad_bias = empty + + return grad_input, grad_bias + + @staticmethod + def backward(ctx, gradgrad_input, gradgrad_bias): + out, = ctx.saved_tensors + gradgrad_out = fused.fused_bias_act( + gradgrad_input.contiguous(), + gradgrad_bias, + out, + 3, + 1, + ctx.negative_slope, + ctx.scale, + ) + + return gradgrad_out, None, None, None, None + + +class FusedLeakyReLUFunction(Function): + @staticmethod + def forward(ctx, input, bias, negative_slope, scale): + empty = input.new_empty(0) + + ctx.bias = bias is not None + + if bias is None: + bias = empty + + out = fused.fused_bias_act(input, bias, empty, 3, 0, negative_slope, scale) + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + return out + + @staticmethod + def backward(ctx, grad_output): + out, = ctx.saved_tensors + + grad_input, grad_bias = FusedLeakyReLUFunctionBackward.apply( + grad_output, out, ctx.bias, ctx.negative_slope, ctx.scale + ) + + if not ctx.bias: + grad_bias = None + + return grad_input, grad_bias, None, None + + +class FusedLeakyReLU(nn.Module): + def __init__(self, channel, bias=True, negative_slope=0.2, scale=2 ** 0.5): + super().__init__() + + if bias: + self.bias = nn.Parameter(torch.zeros(channel)) + + else: + self.bias = None + + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + return fused_leaky_relu(input, self.bias, self.negative_slope, self.scale) + + +def fused_leaky_relu(input, bias=None, negative_slope=0.2, scale=2 ** 0.5): + if input.device.type == "cpu": + if bias is not None: + rest_dim = [1] * (input.ndim - bias.ndim - 1) + return ( + F.leaky_relu( + input + bias.view(1, bias.shape[0], *rest_dim), negative_slope=0.2 + ) + * scale + ) + + else: + return F.leaky_relu(input, negative_slope=0.2) * scale + + else: + return FusedLeakyReLUFunction.apply( + input.contiguous(), bias, negative_slope, scale + ) diff --git a/op/fused_bias_act.cpp b/op/fused_bias_act.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e8466aa15b2a36d501eda423ccb31b7b9cbd03e5 --- /dev/null +++ b/op/fused_bias_act.cpp @@ -0,0 +1,32 @@ + +#include +#include + +torch::Tensor fused_bias_act_op(const torch::Tensor &input, + const torch::Tensor &bias, + const torch::Tensor &refer, int act, int grad, + float alpha, float scale); + +#define CHECK_CUDA(x) \ + TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) + +torch::Tensor fused_bias_act(const torch::Tensor &input, + const torch::Tensor &bias, + const torch::Tensor &refer, int act, int grad, + float alpha, float scale) { + CHECK_INPUT(input); + CHECK_INPUT(bias); + + at::DeviceGuard guard(input.device()); + + return fused_bias_act_op(input, bias, refer, act, grad, alpha, scale); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("fused_bias_act", &fused_bias_act, "fused bias act (CUDA)"); +} \ No newline at end of file diff --git a/op/fused_bias_act_kernel.cu b/op/fused_bias_act_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..65427d333b8a615d1cea5765de22318623706134 --- /dev/null +++ b/op/fused_bias_act_kernel.cu @@ -0,0 +1,105 @@ +// Copyright (c) 2019, NVIDIA Corporation. All rights reserved. +// +// This work is made available under the Nvidia Source Code License-NC. +// To view a copy of this license, visit +// https://nvlabs.github.io/stylegan2/license.html + +#include + +#include +#include +#include +#include + + +#include +#include + +template +static __global__ void +fused_bias_act_kernel(scalar_t *out, const scalar_t *p_x, const scalar_t *p_b, + const scalar_t *p_ref, int act, int grad, scalar_t alpha, + scalar_t scale, int loop_x, int size_x, int step_b, + int size_b, int use_bias, int use_ref) { + int xi = blockIdx.x * loop_x * blockDim.x + threadIdx.x; + + scalar_t zero = 0.0; + + for (int loop_idx = 0; loop_idx < loop_x && xi < size_x; + loop_idx++, xi += blockDim.x) { + scalar_t x = p_x[xi]; + + if (use_bias) { + x += p_b[(xi / step_b) % size_b]; + } + + scalar_t ref = use_ref ? p_ref[xi] : zero; + + scalar_t y; + + switch (act * 10 + grad) { + default: + case 10: + y = x; + break; + case 11: + y = x; + break; + case 12: + y = 0.0; + break; + + case 30: + y = (x > 0.0) ? x : x * alpha; + break; + case 31: + y = (ref > 0.0) ? x : x * alpha; + break; + case 32: + y = 0.0; + break; + } + + out[xi] = y * scale; + } +} + +torch::Tensor fused_bias_act_op(const torch::Tensor &input, + const torch::Tensor &bias, + const torch::Tensor &refer, int act, int grad, + float alpha, float scale) { + int curDevice = -1; + cudaGetDevice(&curDevice); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + auto x = input.contiguous(); + auto b = bias.contiguous(); + auto ref = refer.contiguous(); + + int use_bias = b.numel() ? 1 : 0; + int use_ref = ref.numel() ? 1 : 0; + + int size_x = x.numel(); + int size_b = b.numel(); + int step_b = 1; + + for (int i = 1 + 1; i < x.dim(); i++) { + step_b *= x.size(i); + } + + int loop_x = 4; + int block_size = 4 * 32; + int grid_size = (size_x - 1) / (loop_x * block_size) + 1; + + auto y = torch::empty_like(x); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + x.scalar_type(), "fused_bias_act_kernel", [&] { + fused_bias_act_kernel<<>>( + y.data_ptr(), x.data_ptr(), + b.data_ptr(), ref.data_ptr(), act, grad, alpha, + scale, loop_x, size_x, step_b, size_b, use_bias, use_ref); + }); + + return y; +} \ No newline at end of file diff --git a/op/upfirdn2d.cpp b/op/upfirdn2d.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d667b2cfcb2eb129b6582dea58dc0dda31e4bc45 --- /dev/null +++ b/op/upfirdn2d.cpp @@ -0,0 +1,31 @@ +#include +#include + +torch::Tensor upfirdn2d_op(const torch::Tensor &input, + const torch::Tensor &kernel, int up_x, int up_y, + int down_x, int down_y, int pad_x0, int pad_x1, + int pad_y0, int pad_y1); + +#define CHECK_CUDA(x) \ + TORCH_CHECK(x.type().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) + +torch::Tensor upfirdn2d(const torch::Tensor &input, const torch::Tensor &kernel, + int up_x, int up_y, int down_x, int down_y, int pad_x0, + int pad_x1, int pad_y0, int pad_y1) { + CHECK_INPUT(input); + CHECK_INPUT(kernel); + + at::DeviceGuard guard(input.device()); + + return upfirdn2d_op(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, + pad_y0, pad_y1); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("upfirdn2d", &upfirdn2d, "upfirdn2d (CUDA)"); +} \ No newline at end of file diff --git a/op/upfirdn2d.py b/op/upfirdn2d.py new file mode 100644 index 0000000000000000000000000000000000000000..67e0375efac0f1cadcda12e667b72c36369e9c88 --- /dev/null +++ b/op/upfirdn2d.py @@ -0,0 +1,209 @@ +from collections import abc +import os + +import torch +from torch.nn import functional as F +from torch.autograd import Function +from torch.utils.cpp_extension import load + + +module_path = os.path.dirname(__file__) +upfirdn2d_op = load( + "upfirdn2d", + sources=[ + os.path.join(module_path, "upfirdn2d.cpp"), + os.path.join(module_path, "upfirdn2d_kernel.cu"), + ], +) + + +class UpFirDn2dBackward(Function): + @staticmethod + def forward( + ctx, grad_output, kernel, grad_kernel, up, down, pad, g_pad, in_size, out_size + ): + + up_x, up_y = up + down_x, down_y = down + g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1 = g_pad + + grad_output = grad_output.reshape(-1, out_size[0], out_size[1], 1) + + grad_input = upfirdn2d_op.upfirdn2d( + grad_output, + grad_kernel, + down_x, + down_y, + up_x, + up_y, + g_pad_x0, + g_pad_x1, + g_pad_y0, + g_pad_y1, + ) + grad_input = grad_input.view(in_size[0], in_size[1], in_size[2], in_size[3]) + + ctx.save_for_backward(kernel) + + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + ctx.up_x = up_x + ctx.up_y = up_y + ctx.down_x = down_x + ctx.down_y = down_y + ctx.pad_x0 = pad_x0 + ctx.pad_x1 = pad_x1 + ctx.pad_y0 = pad_y0 + ctx.pad_y1 = pad_y1 + ctx.in_size = in_size + ctx.out_size = out_size + + return grad_input + + @staticmethod + def backward(ctx, gradgrad_input): + kernel, = ctx.saved_tensors + + gradgrad_input = gradgrad_input.reshape(-1, ctx.in_size[2], ctx.in_size[3], 1) + + gradgrad_out = upfirdn2d_op.upfirdn2d( + gradgrad_input, + kernel, + ctx.up_x, + ctx.up_y, + ctx.down_x, + ctx.down_y, + ctx.pad_x0, + ctx.pad_x1, + ctx.pad_y0, + ctx.pad_y1, + ) + # gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.out_size[0], ctx.out_size[1], ctx.in_size[3]) + gradgrad_out = gradgrad_out.view( + ctx.in_size[0], ctx.in_size[1], ctx.out_size[0], ctx.out_size[1] + ) + + return gradgrad_out, None, None, None, None, None, None, None, None + + +class UpFirDn2d(Function): + @staticmethod + def forward(ctx, input, kernel, up, down, pad): + up_x, up_y = up + down_x, down_y = down + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + kernel_h, kernel_w = kernel.shape + batch, channel, in_h, in_w = input.shape + ctx.in_size = input.shape + + input = input.reshape(-1, in_h, in_w, 1) + + ctx.save_for_backward(kernel, torch.flip(kernel, [0, 1])) + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + ctx.out_size = (out_h, out_w) + + ctx.up = (up_x, up_y) + ctx.down = (down_x, down_y) + ctx.pad = (pad_x0, pad_x1, pad_y0, pad_y1) + + g_pad_x0 = kernel_w - pad_x0 - 1 + g_pad_y0 = kernel_h - pad_y0 - 1 + g_pad_x1 = in_w * up_x - out_w * down_x + pad_x0 - up_x + 1 + g_pad_y1 = in_h * up_y - out_h * down_y + pad_y0 - up_y + 1 + + ctx.g_pad = (g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1) + + out = upfirdn2d_op.upfirdn2d( + input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1 + ) + # out = out.view(major, out_h, out_w, minor) + out = out.view(-1, channel, out_h, out_w) + + return out + + @staticmethod + def backward(ctx, grad_output): + kernel, grad_kernel = ctx.saved_tensors + + grad_input = None + + if ctx.needs_input_grad[0]: + grad_input = UpFirDn2dBackward.apply( + grad_output, + kernel, + grad_kernel, + ctx.up, + ctx.down, + ctx.pad, + ctx.g_pad, + ctx.in_size, + ctx.out_size, + ) + + return grad_input, None, None, None, None + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + if not isinstance(up, abc.Iterable): + up = (up, up) + + if not isinstance(down, abc.Iterable): + down = (down, down) + + if len(pad) == 2: + pad = (pad[0], pad[1], pad[0], pad[1]) + + if input.device.type == "cpu": + out = upfirdn2d_native(input, kernel, *up, *down, *pad) + + else: + out = UpFirDn2d.apply(input, kernel, up, down, pad) + + return out + + +def upfirdn2d_native( + input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1 +): + _, channel, in_h, in_w = input.shape + input = input.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad( + out, [0, 0, max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)] + ) + out = out[ + :, + max(-pad_y0, 0) : out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0) : out.shape[2] - max(-pad_x1, 0), + :, + ] + + out = out.permute(0, 3, 1, 2) + out = out.reshape( + [-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1] + ) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h + down_y) // down_y + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w + down_x) // down_x + + return out.view(-1, channel, out_h, out_w) diff --git a/op/upfirdn2d_kernel.cu b/op/upfirdn2d_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..61d3dc413041738bb6885e33da7b3ed8fbfa2496 --- /dev/null +++ b/op/upfirdn2d_kernel.cu @@ -0,0 +1,369 @@ +// Copyright (c) 2019, NVIDIA Corporation. All rights reserved. +// +// This work is made available under the Nvidia Source Code License-NC. +// To view a copy of this license, visit +// https://nvlabs.github.io/stylegan2/license.html + +#include + +#include +#include +#include +#include + +#include +#include + +static __host__ __device__ __forceinline__ int floor_div(int a, int b) { + int c = a / b; + + if (c * b > a) { + c--; + } + + return c; +} + +struct UpFirDn2DKernelParams { + int up_x; + int up_y; + int down_x; + int down_y; + int pad_x0; + int pad_x1; + int pad_y0; + int pad_y1; + + int major_dim; + int in_h; + int in_w; + int minor_dim; + int kernel_h; + int kernel_w; + int out_h; + int out_w; + int loop_major; + int loop_x; +}; + +template +__global__ void upfirdn2d_kernel_large(scalar_t *out, const scalar_t *input, + const scalar_t *kernel, + const UpFirDn2DKernelParams p) { + int minor_idx = blockIdx.x * blockDim.x + threadIdx.x; + int out_y = minor_idx / p.minor_dim; + minor_idx -= out_y * p.minor_dim; + int out_x_base = blockIdx.y * p.loop_x * blockDim.y + threadIdx.y; + int major_idx_base = blockIdx.z * p.loop_major; + + if (out_x_base >= p.out_w || out_y >= p.out_h || + major_idx_base >= p.major_dim) { + return; + } + + int mid_y = out_y * p.down_y + p.up_y - 1 - p.pad_y0; + int in_y = min(max(floor_div(mid_y, p.up_y), 0), p.in_h); + int h = min(max(floor_div(mid_y + p.kernel_h, p.up_y), 0), p.in_h) - in_y; + int kernel_y = mid_y + p.kernel_h - (in_y + 1) * p.up_y; + + for (int loop_major = 0, major_idx = major_idx_base; + loop_major < p.loop_major && major_idx < p.major_dim; + loop_major++, major_idx++) { + for (int loop_x = 0, out_x = out_x_base; + loop_x < p.loop_x && out_x < p.out_w; loop_x++, out_x += blockDim.y) { + int mid_x = out_x * p.down_x + p.up_x - 1 - p.pad_x0; + int in_x = min(max(floor_div(mid_x, p.up_x), 0), p.in_w); + int w = min(max(floor_div(mid_x + p.kernel_w, p.up_x), 0), p.in_w) - in_x; + int kernel_x = mid_x + p.kernel_w - (in_x + 1) * p.up_x; + + const scalar_t *x_p = + &input[((major_idx * p.in_h + in_y) * p.in_w + in_x) * p.minor_dim + + minor_idx]; + const scalar_t *k_p = &kernel[kernel_y * p.kernel_w + kernel_x]; + int x_px = p.minor_dim; + int k_px = -p.up_x; + int x_py = p.in_w * p.minor_dim; + int k_py = -p.up_y * p.kernel_w; + + scalar_t v = 0.0f; + + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + v += static_cast(*x_p) * static_cast(*k_p); + x_p += x_px; + k_p += k_px; + } + + x_p += x_py - w * x_px; + k_p += k_py - w * k_px; + } + + out[((major_idx * p.out_h + out_y) * p.out_w + out_x) * p.minor_dim + + minor_idx] = v; + } + } +} + +template +__global__ void upfirdn2d_kernel(scalar_t *out, const scalar_t *input, + const scalar_t *kernel, + const UpFirDn2DKernelParams p) { + const int tile_in_h = ((tile_out_h - 1) * down_y + kernel_h - 1) / up_y + 1; + const int tile_in_w = ((tile_out_w - 1) * down_x + kernel_w - 1) / up_x + 1; + + __shared__ volatile float sk[kernel_h][kernel_w]; + __shared__ volatile float sx[tile_in_h][tile_in_w]; + + int minor_idx = blockIdx.x; + int tile_out_y = minor_idx / p.minor_dim; + minor_idx -= tile_out_y * p.minor_dim; + tile_out_y *= tile_out_h; + int tile_out_x_base = blockIdx.y * p.loop_x * tile_out_w; + int major_idx_base = blockIdx.z * p.loop_major; + + if (tile_out_x_base >= p.out_w | tile_out_y >= p.out_h | + major_idx_base >= p.major_dim) { + return; + } + + for (int tap_idx = threadIdx.x; tap_idx < kernel_h * kernel_w; + tap_idx += blockDim.x) { + int ky = tap_idx / kernel_w; + int kx = tap_idx - ky * kernel_w; + scalar_t v = 0.0; + + if (kx < p.kernel_w & ky < p.kernel_h) { + v = kernel[(p.kernel_h - 1 - ky) * p.kernel_w + (p.kernel_w - 1 - kx)]; + } + + sk[ky][kx] = v; + } + + for (int loop_major = 0, major_idx = major_idx_base; + loop_major < p.loop_major & major_idx < p.major_dim; + loop_major++, major_idx++) { + for (int loop_x = 0, tile_out_x = tile_out_x_base; + loop_x < p.loop_x & tile_out_x < p.out_w; + loop_x++, tile_out_x += tile_out_w) { + int tile_mid_x = tile_out_x * down_x + up_x - 1 - p.pad_x0; + int tile_mid_y = tile_out_y * down_y + up_y - 1 - p.pad_y0; + int tile_in_x = floor_div(tile_mid_x, up_x); + int tile_in_y = floor_div(tile_mid_y, up_y); + + __syncthreads(); + + for (int in_idx = threadIdx.x; in_idx < tile_in_h * tile_in_w; + in_idx += blockDim.x) { + int rel_in_y = in_idx / tile_in_w; + int rel_in_x = in_idx - rel_in_y * tile_in_w; + int in_x = rel_in_x + tile_in_x; + int in_y = rel_in_y + tile_in_y; + + scalar_t v = 0.0; + + if (in_x >= 0 & in_y >= 0 & in_x < p.in_w & in_y < p.in_h) { + v = input[((major_idx * p.in_h + in_y) * p.in_w + in_x) * + p.minor_dim + + minor_idx]; + } + + sx[rel_in_y][rel_in_x] = v; + } + + __syncthreads(); + for (int out_idx = threadIdx.x; out_idx < tile_out_h * tile_out_w; + out_idx += blockDim.x) { + int rel_out_y = out_idx / tile_out_w; + int rel_out_x = out_idx - rel_out_y * tile_out_w; + int out_x = rel_out_x + tile_out_x; + int out_y = rel_out_y + tile_out_y; + + int mid_x = tile_mid_x + rel_out_x * down_x; + int mid_y = tile_mid_y + rel_out_y * down_y; + int in_x = floor_div(mid_x, up_x); + int in_y = floor_div(mid_y, up_y); + int rel_in_x = in_x - tile_in_x; + int rel_in_y = in_y - tile_in_y; + int kernel_x = (in_x + 1) * up_x - mid_x - 1; + int kernel_y = (in_y + 1) * up_y - mid_y - 1; + + scalar_t v = 0.0; + +#pragma unroll + for (int y = 0; y < kernel_h / up_y; y++) +#pragma unroll + for (int x = 0; x < kernel_w / up_x; x++) + v += sx[rel_in_y + y][rel_in_x + x] * + sk[kernel_y + y * up_y][kernel_x + x * up_x]; + + if (out_x < p.out_w & out_y < p.out_h) { + out[((major_idx * p.out_h + out_y) * p.out_w + out_x) * p.minor_dim + + minor_idx] = v; + } + } + } + } +} + +torch::Tensor upfirdn2d_op(const torch::Tensor &input, + const torch::Tensor &kernel, int up_x, int up_y, + int down_x, int down_y, int pad_x0, int pad_x1, + int pad_y0, int pad_y1) { + int curDevice = -1; + cudaGetDevice(&curDevice); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + UpFirDn2DKernelParams p; + + auto x = input.contiguous(); + auto k = kernel.contiguous(); + + p.major_dim = x.size(0); + p.in_h = x.size(1); + p.in_w = x.size(2); + p.minor_dim = x.size(3); + p.kernel_h = k.size(0); + p.kernel_w = k.size(1); + p.up_x = up_x; + p.up_y = up_y; + p.down_x = down_x; + p.down_y = down_y; + p.pad_x0 = pad_x0; + p.pad_x1 = pad_x1; + p.pad_y0 = pad_y0; + p.pad_y1 = pad_y1; + + p.out_h = (p.in_h * p.up_y + p.pad_y0 + p.pad_y1 - p.kernel_h + p.down_y) / + p.down_y; + p.out_w = (p.in_w * p.up_x + p.pad_x0 + p.pad_x1 - p.kernel_w + p.down_x) / + p.down_x; + + auto out = + at::empty({p.major_dim, p.out_h, p.out_w, p.minor_dim}, x.options()); + + int mode = -1; + + int tile_out_h = -1; + int tile_out_w = -1; + + if (p.up_x == 1 && p.up_y == 1 && p.down_x == 1 && p.down_y == 1 && + p.kernel_h <= 4 && p.kernel_w <= 4) { + mode = 1; + tile_out_h = 16; + tile_out_w = 64; + } + + if (p.up_x == 1 && p.up_y == 1 && p.down_x == 1 && p.down_y == 1 && + p.kernel_h <= 3 && p.kernel_w <= 3) { + mode = 2; + tile_out_h = 16; + tile_out_w = 64; + } + + if (p.up_x == 2 && p.up_y == 2 && p.down_x == 1 && p.down_y == 1 && + p.kernel_h <= 4 && p.kernel_w <= 4) { + mode = 3; + tile_out_h = 16; + tile_out_w = 64; + } + + if (p.up_x == 2 && p.up_y == 2 && p.down_x == 1 && p.down_y == 1 && + p.kernel_h <= 2 && p.kernel_w <= 2) { + mode = 4; + tile_out_h = 16; + tile_out_w = 64; + } + + if (p.up_x == 1 && p.up_y == 1 && p.down_x == 2 && p.down_y == 2 && + p.kernel_h <= 4 && p.kernel_w <= 4) { + mode = 5; + tile_out_h = 8; + tile_out_w = 32; + } + + if (p.up_x == 1 && p.up_y == 1 && p.down_x == 2 && p.down_y == 2 && + p.kernel_h <= 2 && p.kernel_w <= 2) { + mode = 6; + tile_out_h = 8; + tile_out_w = 32; + } + + dim3 block_size; + dim3 grid_size; + + if (tile_out_h > 0 && tile_out_w > 0) { + p.loop_major = (p.major_dim - 1) / 16384 + 1; + p.loop_x = 1; + block_size = dim3(32 * 8, 1, 1); + grid_size = dim3(((p.out_h - 1) / tile_out_h + 1) * p.minor_dim, + (p.out_w - 1) / (p.loop_x * tile_out_w) + 1, + (p.major_dim - 1) / p.loop_major + 1); + } else { + p.loop_major = (p.major_dim - 1) / 16384 + 1; + p.loop_x = 4; + block_size = dim3(4, 32, 1); + grid_size = dim3((p.out_h * p.minor_dim - 1) / block_size.x + 1, + (p.out_w - 1) / (p.loop_x * block_size.y) + 1, + (p.major_dim - 1) / p.loop_major + 1); + } + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(x.scalar_type(), "upfirdn2d_cuda", [&] { + switch (mode) { + case 1: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + case 2: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + case 3: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + case 4: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + case 5: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + case 6: + upfirdn2d_kernel + <<>>(out.data_ptr(), + x.data_ptr(), + k.data_ptr(), p); + + break; + + default: + upfirdn2d_kernel_large<<>>( + out.data_ptr(), x.data_ptr(), + k.data_ptr(), p); + } + }); + + return out; +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3263f045da25c0c21c061cacc7658d0dfeeb4f0b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +torch +torchvision +numpy +matplotlib +opencv-python +imageio +ninja +einops +opencv-python-headless +datasets +modelscope[framework] +ultralytics +cnstd==1.2.3 + diff --git a/test_marconetplus.py b/test_marconetplus.py new file mode 100644 index 0000000000000000000000000000000000000000..8561b6132c03e6b0ef3550e1acd8a3a2a8969bb7 --- /dev/null +++ b/test_marconetplus.py @@ -0,0 +1,163 @@ + +import torch +import cv2 +import numpy as np +import torch.nn.functional as F +import time +import argparse +import os +import os.path as osp +from models.TextEnhancement import MARCONetPlus +from utils.utils_image import get_image_paths, imread_uint, uint2tensor4, tensor2uint +from networks.rrdbnet2_arch import RRDBNet as BSRGAN + + + + +def inference(input_path=None, output_path=None, aligned=False, bg_sr=False, scale_factor=2, save_text=False, device=None): + + if device == None or device == 'gpu': + use_cuda = torch.cuda.is_available() + if device == 'cpu': + use_cuda = False + device = torch.device('cuda' if use_cuda else 'cpu') + + if input_path is None: + exit('input image path is none. Please see our document') + + if output_path is None: + TIMESTAMP = time.strftime("%m-%d_%H-%M", time.localtime()) + if input_path[-1] == '/' or input_path[-1] == '\\': + input_path = input_path[:-1] + output_path = osp.join(input_path+'_'+TIMESTAMP+'_MARCONetPlus') + os.makedirs(output_path, exist_ok=True) + + # use bsrgan to restore the background of the whole image + if bg_sr: + ##BG model + BGModel = BSRGAN(in_nc=3, out_nc=3, nf=64, nb=23, gc=32, sf=2) # define network + model_old = torch.load('./checkpoints/bsrgan_bg.pth') + state_dict = BGModel.state_dict() + for ((key, param),(key2, param2)) in zip(model_old.items(), state_dict.items()): + state_dict[key2] = param + BGModel.load_state_dict(state_dict, strict=True) + BGModel.eval() + for k, v in BGModel.named_parameters(): + v.requires_grad = False + BGModel = BGModel.to(device) + torch.cuda.empty_cache() + + + lq_paths = get_image_paths(input_path) + if len(lq_paths) ==0: + exit('No Image in the LR path.') + + + WEncoderPath='./checkpoints/net_w_encoder_860000.pth' + PriorModelPath='./checkpoints/net_prior_860000.pth' + SRModelPath='./checkpoints/net_sr_860000.pth' + YoloPath = './checkpoints/yolo11m_short_character.pt' + + TextModel = MARCONetPlus(WEncoderPath, PriorModelPath, SRModelPath, YoloPath, device=device) + + print('{:>25s} : {:s}'.format('Model Name', 'MARCONetPlusPlus')) + if use_cuda: + print('{:>25s} : {:25s} : {:s}'.format('GPU ID', 'No GPU is available. Use CPU instead.')) + torch.cuda.empty_cache() + + L_path = input_path + E_path = output_path # save path + print('{:>25s} : {:s}'.format('Input Path', L_path)) + print('{:>25s} : {:s}'.format('Output Path', E_path)) + if aligned: + print('{:>25s} : {:s}'.format('Image Details', 'Aligned Text Layout. No text detection is used.')) + else: + print('{:>25s} : {:s}'.format('Image Details', 'UnAligned Text Image. It will crop text region using CnSTD, restore, and paste results back.')) + print('{:>25s} : {}'.format('Scale Facter', scale_factor)) + print('{:>25s} : {:s}'.format('Save LR & SR text layout', 'True' if save_text else 'False')) + + idx = 0 + + for iix, img_path in enumerate(lq_paths): + #################################### + #####(1) Read Image + #################################### + idx += 1 + img_name, ext = os.path.splitext(os.path.basename(img_path)) + print('{:>20s} {:04d} --> {: 0: + mean_color = cv2.mean(segment_img, mask=non_text_mask)[:3] + mean_color = np.array(mean_color, dtype=np.uint8) + else: + mean_color = np.array([255, 255, 255], dtype=np.uint8) + mean_img = np.full(segment_img.shape, mean_color, dtype=np.uint8) + blurred_mask = cv2.GaussianBlur(mask, (15, 15), 0) + alpha = blurred_mask.astype(np.float32) / 255.0 + alpha = np.expand_dims(alpha, axis=2) + segment_img_masked = (segment_img * alpha + mean_img * (1 - alpha)).astype(np.uint8) + ocr_result = ocr_pipeline(segment_img_masked) + segment_text = ocr_result['text'][0] if 'text' in ocr_result else '' + segment_text = segment_text.replace(' ', '') + if len(segment_text) == num_cropped_boxes: + char = segment_text[j - idxs[0]] + elif len(segment_text) > 0: + char = segment_text[min(j - idxs[0], len(segment_text)-1)] + else: + char = '' + recognized_chars.append(char) + x1, _, x2, _ = box + x_center = (x1 + x2) // 2 + char_x_centers.append(x_center) + + # if img.ndim == 2: + # img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) # GGG + # else: + # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # RGB + return boxes, recognized_chars, char_x_centers \ No newline at end of file