init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,29 @@
+---
+tags:
+- image-to-text
+- image-captioning
+language:
+- th
+---
+# Thai Image Captioning
+Encoder-decoder style image captioning model using [CLIP encoder](https://huggingface.co/openai/clip-vit-base-patch32) and [Wangchanberta](https://huggingface.co/airesearch/wangchanberta-base-att-spm-uncased). Trained on Thai language MSCOCO and IPU24 dataset.
+
+# Usage
+
+Use `AutoModel` to load it. Requires `trust_remote_code=True`.
+```python
+from transformers import AutoModel, AutoImageProcessor, AutoTokenizer
+device = 'cuda'
+gen_kwargs = {"max_length": 120, "num_beams": 4}
+model_path = 'Natthaphon/thaicapgen-clip-gpt2'
+feature_extractor = AutoImageProcessor.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModel.from_pretrained(model_path, trust_remote_code=True).to(device)
+pixel_values = feature_extractor(images=[Image.open(image_path)], return_tensors="pt").pixel_values
+pixel_values = pixel_values.to(device)
+output_ids = model.generate(pixel_values, **gen_kwargs)
+preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
+```
+
+# Acknowledgement
+This work is partially supported by the Program Management Unit for Human Resources & Institutional Development, Research and Innovation (PMU-B) [Grant number B04G640107]

config.json

@@ -0,0 +1,2059 @@
+{
+  "_name_or_path": "/media/palm/BiggerData/capgen/hub/pth/wangchan_clip_1e-05_encoder_freeze",
+  "architectures": [
+    "CLIPEncoderDecoderModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_clipcap.CLIPEncoderDecoderConfig",
+    "AutoModel": "modeling_clipcap.CLIPEncoderDecoderModel"
+  },
+  "decoder": {
+    "_name_or_path": "airesearch/wangchanberta-base-att-spm-uncased",
+    "add_cross_attention": true,
+    "architectures": [
+      "RobertaForMaskedLM"
+    ],
+    "gradient_checkpointing": false,
+    "is_decoder": true,
+    "model_type": "camembert",
+    "num_attention_head": 12,
+    "type_vocab_size": 1,
+    "vocab_size": 25005
+  },
+  "decoder_start_token_id": 5,
+  "encoder": {
+    "_name_or_path": "microsoft/swinv2-large-patch4-window12to16-192to256-22kto1k-ft",
+    "architectures": [
+      "Swinv2ForImageClassification"
+    ],
+    "depths": [
+      2,
+      2,
+      18,
+      2
+    ],
+    "embed_dim": 192,
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "tench, Tinca tinca",
+      "1": "goldfish, Carassius auratus",
+      "2": "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias",
+      "3": "tiger shark, Galeocerdo cuvieri",
+      "4": "hammerhead, hammerhead shark",
+      "5": "electric ray, crampfish, numbfish, torpedo",
+      "6": "stingray",
+      "7": "cock",
+      "8": "hen",
+      "9": "ostrich, Struthio camelus",
+      "10": "brambling, Fringilla montifringilla",
+      "11": "goldfinch, Carduelis carduelis",
+      "12": "house finch, linnet, Carpodacus mexicanus",
+      "13": "junco, snowbird",
+      "14": "indigo bunting, indigo finch, indigo bird, Passerina cyanea",
+      "15": "robin, American robin, Turdus migratorius",
+      "16": "bulbul",
+      "17": "jay",
+      "18": "magpie",
+      "19": "chickadee",
+      "20": "water ouzel, dipper",
+      "21": "kite",
+      "22": "bald eagle, American eagle, Haliaeetus leucocephalus",
+      "23": "vulture",
+      "24": "great grey owl, great gray owl, Strix nebulosa",
+      "25": "European fire salamander, Salamandra salamandra",
+      "26": "common newt, Triturus vulgaris",
+      "27": "eft",
+      "28": "spotted salamander, Ambystoma maculatum",
+      "29": "axolotl, mud puppy, Ambystoma mexicanum",
+      "30": "bullfrog, Rana catesbeiana",
+      "31": "tree frog, tree-frog",
+      "32": "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui",
+      "33": "loggerhead, loggerhead turtle, Caretta caretta",
+      "34": "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea",
+      "35": "mud turtle",
+      "36": "terrapin",
+      "37": "box turtle, box tortoise",
+      "38": "banded gecko",
+      "39": "common iguana, iguana, Iguana iguana",
+      "40": "American chameleon, anole, Anolis carolinensis",
+      "41": "whiptail, whiptail lizard",
+      "42": "agama",
+      "43": "frilled lizard, Chlamydosaurus kingi",
+      "44": "alligator lizard",
+      "45": "Gila monster, Heloderma suspectum",
+      "46": "green lizard, Lacerta viridis",
+      "47": "African chameleon, Chamaeleo chamaeleon",
+      "48": "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis",
+      "49": "African crocodile, Nile crocodile, Crocodylus niloticus",
+      "50": "American alligator, Alligator mississipiensis",
+      "51": "triceratops",
+      "52": "thunder snake, worm snake, Carphophis amoenus",
+      "53": "ringneck snake, ring-necked snake, ring snake",
+      "54": "hognose snake, puff adder, sand viper",
+      "55": "green snake, grass snake",
+      "56": "king snake, kingsnake",
+      "57": "garter snake, grass snake",
+      "58": "water snake",
+      "59": "vine snake",
+      "60": "night snake, Hypsiglena torquata",
+      "61": "boa constrictor, Constrictor constrictor",
+      "62": "rock python, rock snake, Python sebae",
+      "63": "Indian cobra, Naja naja",
+      "64": "green mamba",
+      "65": "sea snake",
+      "66": "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus",
+      "67": "diamondback, diamondback rattlesnake, Crotalus adamanteus",
+      "68": "sidewinder, horned rattlesnake, Crotalus cerastes",
+      "69": "trilobite",
+      "70": "harvestman, daddy longlegs, Phalangium opilio",
+      "71": "scorpion",
+      "72": "black and gold garden spider, Argiope aurantia",
+      "73": "barn spider, Araneus cavaticus",
+      "74": "garden spider, Aranea diademata",
+      "75": "black widow, Latrodectus mactans",
+      "76": "tarantula",
+      "77": "wolf spider, hunting spider",
+      "78": "tick",
+      "79": "centipede",
+      "80": "black grouse",
+      "81": "ptarmigan",
+      "82": "ruffed grouse, partridge, Bonasa umbellus",
+      "83": "prairie chicken, prairie grouse, prairie fowl",
+      "84": "peacock",
+      "85": "quail",
+      "86": "partridge",
+      "87": "African grey, African gray, Psittacus erithacus",
+      "88": "macaw",
+      "89": "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita",
+      "90": "lorikeet",
+      "91": "coucal",
+      "92": "bee eater",
+      "93": "hornbill",
+      "94": "hummingbird",
+      "95": "jacamar",
+      "96": "toucan",
+      "97": "drake",
+      "98": "red-breasted merganser, Mergus serrator",
+      "99": "goose",
+      "100": "black swan, Cygnus atratus",
+      "101": "tusker",
+      "102": "echidna, spiny anteater, anteater",
+      "103": "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus",
+      "104": "wallaby, brush kangaroo",
+      "105": "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus",
+      "106": "wombat",
+      "107": "jellyfish",
+      "108": "sea anemone, anemone",
+      "109": "brain coral",
+      "110": "flatworm, platyhelminth",
+      "111": "nematode, nematode worm, roundworm",
+      "112": "conch",
+      "113": "snail",
+      "114": "slug",
+      "115": "sea slug, nudibranch",
+      "116": "chiton, coat-of-mail shell, sea cradle, polyplacophore",
+      "117": "chambered nautilus, pearly nautilus, nautilus",
+      "118": "Dungeness crab, Cancer magister",
+      "119": "rock crab, Cancer irroratus",
+      "120": "fiddler crab",
+      "121": "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica",
+      "122": "American lobster, Northern lobster, Maine lobster, Homarus americanus",
+      "123": "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish",
+      "124": "crayfish, crawfish, crawdad, crawdaddy",
+      "125": "hermit crab",
+      "126": "isopod",
+      "127": "white stork, Ciconia ciconia",
+      "128": "black stork, Ciconia nigra",
+      "129": "spoonbill",
+      "130": "flamingo",
+      "131": "little blue heron, Egretta caerulea",
+      "132": "American egret, great white heron, Egretta albus",
+      "133": "bittern",
+      "134": "crane",
+      "135": "limpkin, Aramus pictus",
+      "136": "European gallinule, Porphyrio porphyrio",
+      "137": "American coot, marsh hen, mud hen, water hen, Fulica americana",
+      "138": "bustard",
+      "139": "ruddy turnstone, Arenaria interpres",
+      "140": "red-backed sandpiper, dunlin, Erolia alpina",
+      "141": "redshank, Tringa totanus",
+      "142": "dowitcher",
+      "143": "oystercatcher, oyster catcher",
+      "144": "pelican",
+      "145": "king penguin, Aptenodytes patagonica",
+      "146": "albatross, mollymawk",
+      "147": "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus",
+      "148": "killer whale, killer, orca, grampus, sea wolf, Orcinus orca",
+      "149": "dugong, Dugong dugon",
+      "150": "sea lion",
+      "151": "Chihuahua",
+      "152": "Japanese spaniel",
+      "153": "Maltese dog, Maltese terrier, Maltese",
+      "154": "Pekinese, Pekingese, Peke",
+      "155": "Shih-Tzu",
+      "156": "Blenheim spaniel",
+      "157": "papillon",
+      "158": "toy terrier",
+      "159": "Rhodesian ridgeback",
+      "160": "Afghan hound, Afghan",
+      "161": "basset, basset hound",
+      "162": "beagle",
+      "163": "bloodhound, sleuthhound",
+      "164": "bluetick",
+      "165": "black-and-tan coonhound",
+      "166": "Walker hound, Walker foxhound",
+      "167": "English foxhound",
+      "168": "redbone",
+      "169": "borzoi, Russian wolfhound",
+      "170": "Irish wolfhound",
+      "171": "Italian greyhound",
+      "172": "whippet",
+      "173": "Ibizan hound, Ibizan Podenco",
+      "174": "Norwegian elkhound, elkhound",
+      "175": "otterhound, otter hound",
+      "176": "Saluki, gazelle hound",
+      "177": "Scottish deerhound, deerhound",
+      "178": "Weimaraner",
+      "179": "Staffordshire bullterrier, Staffordshire bull terrier",
+      "180": "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier",
+      "181": "Bedlington terrier",
+      "182": "Border terrier",
+      "183": "Kerry blue terrier",
+      "184": "Irish terrier",
+      "185": "Norfolk terrier",
+      "186": "Norwich terrier",
+      "187": "Yorkshire terrier",
+      "188": "wire-haired fox terrier",
+      "189": "Lakeland terrier",
+      "190": "Sealyham terrier, Sealyham",
+      "191": "Airedale, Airedale terrier",
+      "192": "cairn, cairn terrier",
+      "193": "Australian terrier",
+      "194": "Dandie Dinmont, Dandie Dinmont terrier",
+      "195": "Boston bull, Boston terrier",
+      "196": "miniature schnauzer",
+      "197": "giant schnauzer",
+      "198": "standard schnauzer",
+      "199": "Scotch terrier, Scottish terrier, Scottie",
+      "200": "Tibetan terrier, chrysanthemum dog",
+      "201": "silky terrier, Sydney silky",
+      "202": "soft-coated wheaten terrier",
+      "203": "West Highland white terrier",
+      "204": "Lhasa, Lhasa apso",
+      "205": "flat-coated retriever",
+      "206": "curly-coated retriever",
+      "207": "golden retriever",
+      "208": "Labrador retriever",
+      "209": "Chesapeake Bay retriever",
+      "210": "German short-haired pointer",
+      "211": "vizsla, Hungarian pointer",
+      "212": "English setter",
+      "213": "Irish setter, red setter",
+      "214": "Gordon setter",
+      "215": "Brittany spaniel",
+      "216": "clumber, clumber spaniel",
+      "217": "English springer, English springer spaniel",
+      "218": "Welsh springer spaniel",
+      "219": "cocker spaniel, English cocker spaniel, cocker",
+      "220": "Sussex spaniel",
+      "221": "Irish water spaniel",
+      "222": "kuvasz",
+      "223": "schipperke",
+      "224": "groenendael",
+      "225": "malinois",
+      "226": "briard",
+      "227": "kelpie",
+      "228": "komondor",
+      "229": "Old English sheepdog, bobtail",
+      "230": "Shetland sheepdog, Shetland sheep dog, Shetland",
+      "231": "collie",
+      "232": "Border collie",
+      "233": "Bouvier des Flandres, Bouviers des Flandres",
+      "234": "Rottweiler",
+      "235": "German shepherd, German shepherd dog, German police dog, alsatian",
+      "236": "Doberman, Doberman pinscher",
+      "237": "miniature pinscher",
+      "238": "Greater Swiss Mountain dog",
+      "239": "Bernese mountain dog",
+      "240": "Appenzeller",
+      "241": "EntleBucher",
+      "242": "boxer",
+      "243": "bull mastiff",
+      "244": "Tibetan mastiff",
+      "245": "French bulldog",
+      "246": "Great Dane",
+      "247": "Saint Bernard, St Bernard",
+      "248": "Eskimo dog, husky",
+      "249": "malamute, malemute, Alaskan malamute",
+      "250": "Siberian husky",
+      "251": "dalmatian, coach dog, carriage dog",
+      "252": "affenpinscher, monkey pinscher, monkey dog",
+      "253": "basenji",
+      "254": "pug, pug-dog",
+      "255": "Leonberg",
+      "256": "Newfoundland, Newfoundland dog",
+      "257": "Great Pyrenees",
+      "258": "Samoyed, Samoyede",
+      "259": "Pomeranian",
+      "260": "chow, chow chow",
+      "261": "keeshond",
+      "262": "Brabancon griffon",
+      "263": "Pembroke, Pembroke Welsh corgi",
+      "264": "Cardigan, Cardigan Welsh corgi",
+      "265": "toy poodle",
+      "266": "miniature poodle",
+      "267": "standard poodle",
+      "268": "Mexican hairless",
+      "269": "timber wolf, grey wolf, gray wolf, Canis lupus",
+      "270": "white wolf, Arctic wolf, Canis lupus tundrarum",
+      "271": "red wolf, maned wolf, Canis rufus, Canis niger",
+      "272": "coyote, prairie wolf, brush wolf, Canis latrans",
+      "273": "dingo, warrigal, warragal, Canis dingo",
+      "274": "dhole, Cuon alpinus",
+      "275": "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus",
+      "276": "hyena, hyaena",
+      "277": "red fox, Vulpes vulpes",
+      "278": "kit fox, Vulpes macrotis",
+      "279": "Arctic fox, white fox, Alopex lagopus",
+      "280": "grey fox, gray fox, Urocyon cinereoargenteus",
+      "281": "tabby, tabby cat",
+      "282": "tiger cat",
+      "283": "Persian cat",
+      "284": "Siamese cat, Siamese",
+      "285": "Egyptian cat",
+      "286": "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor",
+      "287": "lynx, catamount",
+      "288": "leopard, Panthera pardus",
+      "289": "snow leopard, ounce, Panthera uncia",
+      "290": "jaguar, panther, Panthera onca, Felis onca",
+      "291": "lion, king of beasts, Panthera leo",
+      "292": "tiger, Panthera tigris",
+      "293": "cheetah, chetah, Acinonyx jubatus",
+      "294": "brown bear, bruin, Ursus arctos",
+      "295": "American black bear, black bear, Ursus americanus, Euarctos americanus",
+      "296": "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus",
+      "297": "sloth bear, Melursus ursinus, Ursus ursinus",
+      "298": "mongoose",
+      "299": "meerkat, mierkat",
+      "300": "tiger beetle",
+      "301": "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle",
+      "302": "ground beetle, carabid beetle",
+      "303": "long-horned beetle, longicorn, longicorn beetle",
+      "304": "leaf beetle, chrysomelid",
+      "305": "dung beetle",
+      "306": "rhinoceros beetle",
+      "307": "weevil",
+      "308": "fly",
+      "309": "bee",
+      "310": "ant, emmet, pismire",
+      "311": "grasshopper, hopper",
+      "312": "cricket",
+      "313": "walking stick, walkingstick, stick insect",
+      "314": "cockroach, roach",
+      "315": "mantis, mantid",
+      "316": "cicada, cicala",
+      "317": "leafhopper",
+      "318": "lacewing, lacewing fly",
+      "319": "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk",
+      "320": "damselfly",
+      "321": "admiral",
+      "322": "ringlet, ringlet butterfly",
+      "323": "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus",
+      "324": "cabbage butterfly",
+      "325": "sulphur butterfly, sulfur butterfly",
+      "326": "lycaenid, lycaenid butterfly",
+      "327": "starfish, sea star",
+      "328": "sea urchin",
+      "329": "sea cucumber, holothurian",
+      "330": "wood rabbit, cottontail, cottontail rabbit",
+      "331": "hare",
+      "332": "Angora, Angora rabbit",
+      "333": "hamster",
+      "334": "porcupine, hedgehog",
+      "335": "fox squirrel, eastern fox squirrel, Sciurus niger",
+      "336": "marmot",
+      "337": "beaver",
+      "338": "guinea pig, Cavia cobaya",
+      "339": "sorrel",
+      "340": "zebra",
+      "341": "hog, pig, grunter, squealer, Sus scrofa",
+      "342": "wild boar, boar, Sus scrofa",
+      "343": "warthog",
+      "344": "hippopotamus, hippo, river horse, Hippopotamus amphibius",
+      "345": "ox",
+      "346": "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis",
+      "347": "bison",
+      "348": "ram, tup",
+      "349": "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis",
+      "350": "ibex, Capra ibex",
+      "351": "hartebeest",
+      "352": "impala, Aepyceros melampus",
+      "353": "gazelle",
+      "354": "Arabian camel, dromedary, Camelus dromedarius",
+      "355": "llama",
+      "356": "weasel",
+      "357": "mink",
+      "358": "polecat, fitch, foulmart, foumart, Mustela putorius",
+      "359": "black-footed ferret, ferret, Mustela nigripes",
+      "360": "otter",
+      "361": "skunk, polecat, wood pussy",
+      "362": "badger",
+      "363": "armadillo",
+      "364": "three-toed sloth, ai, Bradypus tridactylus",
+      "365": "orangutan, orang, orangutang, Pongo pygmaeus",
+      "366": "gorilla, Gorilla gorilla",
+      "367": "chimpanzee, chimp, Pan troglodytes",
+      "368": "gibbon, Hylobates lar",
+      "369": "siamang, Hylobates syndactylus, Symphalangus syndactylus",
+      "370": "guenon, guenon monkey",
+      "371": "patas, hussar monkey, Erythrocebus patas",
+      "372": "baboon",
+      "373": "macaque",
+      "374": "langur",
+      "375": "colobus, colobus monkey",
+      "376": "proboscis monkey, Nasalis larvatus",
+      "377": "marmoset",
+      "378": "capuchin, ringtail, Cebus capucinus",
+      "379": "howler monkey, howler",
+      "380": "titi, titi monkey",
+      "381": "spider monkey, Ateles geoffroyi",
+      "382": "squirrel monkey, Saimiri sciureus",
+      "383": "Madagascar cat, ring-tailed lemur, Lemur catta",
+      "384": "indri, indris, Indri indri, Indri brevicaudatus",
+      "385": "Indian elephant, Elephas maximus",
+      "386": "African elephant, Loxodonta africana",
+      "387": "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens",
+      "388": "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca",
+      "389": "barracouta, snoek",
+      "390": "eel",
+      "391": "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch",
+      "392": "rock beauty, Holocanthus tricolor",
+      "393": "anemone fish",
+      "394": "sturgeon",
+      "395": "gar, garfish, garpike, billfish, Lepisosteus osseus",
+      "396": "lionfish",
+      "397": "puffer, pufferfish, blowfish, globefish",
+      "398": "abacus",
+      "399": "abaya",
+      "400": "academic gown, academic robe, judge's robe",
+      "401": "accordion, piano accordion, squeeze box",
+      "402": "acoustic guitar",
+      "403": "aircraft carrier, carrier, flattop, attack aircraft carrier",
+      "404": "airliner",
+      "405": "airship, dirigible",
+      "406": "altar",
+      "407": "ambulance",
+      "408": "amphibian, amphibious vehicle",
+      "409": "analog clock",
+      "410": "apiary, bee house",
+      "411": "apron",
+      "412": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin",
+      "413": "assault rifle, assault gun",
+      "414": "backpack, back pack, knapsack, packsack, rucksack, haversack",
+      "415": "bakery, bakeshop, bakehouse",
+      "416": "balance beam, beam",
+      "417": "balloon",
+      "418": "ballpoint, ballpoint pen, ballpen, Biro",
+      "419": "Band Aid",
+      "420": "banjo",
+      "421": "bannister, banister, balustrade, balusters, handrail",
+      "422": "barbell",
+      "423": "barber chair",
+      "424": "barbershop",
+      "425": "barn",
+      "426": "barometer",
+      "427": "barrel, cask",
+      "428": "barrow, garden cart, lawn cart, wheelbarrow",
+      "429": "baseball",
+      "430": "basketball",
+      "431": "bassinet",
+      "432": "bassoon",
+      "433": "bathing cap, swimming cap",
+      "434": "bath towel",
+      "435": "bathtub, bathing tub, bath, tub",
+      "436": "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon",
+      "437": "beacon, lighthouse, beacon light, pharos",
+      "438": "beaker",
+      "439": "bearskin, busby, shako",
+      "440": "beer bottle",
+      "441": "beer glass",
+      "442": "bell cote, bell cot",
+      "443": "bib",
+      "444": "bicycle-built-for-two, tandem bicycle, tandem",
+      "445": "bikini, two-piece",
+      "446": "binder, ring-binder",
+      "447": "binoculars, field glasses, opera glasses",
+      "448": "birdhouse",
+      "449": "boathouse",
+      "450": "bobsled, bobsleigh, bob",
+      "451": "bolo tie, bolo, bola tie, bola",
+      "452": "bonnet, poke bonnet",
+      "453": "bookcase",
+      "454": "bookshop, bookstore, bookstall",
+      "455": "bottlecap",
+      "456": "bow",
+      "457": "bow tie, bow-tie, bowtie",
+      "458": "brass, memorial tablet, plaque",
+      "459": "brassiere, bra, bandeau",
+      "460": "breakwater, groin, groyne, mole, bulwark, seawall, jetty",
+      "461": "breastplate, aegis, egis",
+      "462": "broom",
+      "463": "bucket, pail",
+      "464": "buckle",
+      "465": "bulletproof vest",
+      "466": "bullet train, bullet",
+      "467": "butcher shop, meat market",
+      "468": "cab, hack, taxi, taxicab",
+      "469": "caldron, cauldron",
+      "470": "candle, taper, wax light",
+      "471": "cannon",
+      "472": "canoe",
+      "473": "can opener, tin opener",
+      "474": "cardigan",
+      "475": "car mirror",
+      "476": "carousel, carrousel, merry-go-round, roundabout, whirligig",
+      "477": "carpenter's kit, tool kit",
+      "478": "carton",
+      "479": "car wheel",
+      "480": "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM",
+      "481": "cassette",
+      "482": "cassette player",
+      "483": "castle",
+      "484": "catamaran",
+      "485": "CD player",
+      "486": "cello, violoncello",
+      "487": "cellular telephone, cellular phone, cellphone, cell, mobile phone",
+      "488": "chain",
+      "489": "chainlink fence",
+      "490": "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour",
+      "491": "chain saw, chainsaw",
+      "492": "chest",
+      "493": "chiffonier, commode",
+      "494": "chime, bell, gong",
+      "495": "china cabinet, china closet",
+      "496": "Christmas stocking",
+      "497": "church, church building",
+      "498": "cinema, movie theater, movie theatre, movie house, picture palace",
+      "499": "cleaver, meat cleaver, chopper",
+      "500": "cliff dwelling",
+      "501": "cloak",
+      "502": "clog, geta, patten, sabot",
+      "503": "cocktail shaker",
+      "504": "coffee mug",
+      "505": "coffeepot",
+      "506": "coil, spiral, volute, whorl, helix",
+      "507": "combination lock",
+      "508": "computer keyboard, keypad",
+      "509": "confectionery, confectionary, candy store",
+      "510": "container ship, containership, container vessel",
+      "511": "convertible",
+      "512": "corkscrew, bottle screw",
+      "513": "cornet, horn, trumpet, trump",
+      "514": "cowboy boot",
+      "515": "cowboy hat, ten-gallon hat",
+      "516": "cradle",
+      "517": "crane",
+      "518": "crash helmet",
+      "519": "crate",
+      "520": "crib, cot",
+      "521": "Crock Pot",
+      "522": "croquet ball",
+      "523": "crutch",
+      "524": "cuirass",
+      "525": "dam, dike, dyke",
+      "526": "desk",
+      "527": "desktop computer",
+      "528": "dial telephone, dial phone",
+      "529": "diaper, nappy, napkin",
+      "530": "digital clock",
+      "531": "digital watch",
+      "532": "dining table, board",
+      "533": "dishrag, dishcloth",
+      "534": "dishwasher, dish washer, dishwashing machine",
+      "535": "disk brake, disc brake",
+      "536": "dock, dockage, docking facility",
+      "537": "dogsled, dog sled, dog sleigh",
+      "538": "dome",
+      "539": "doormat, welcome mat",
+      "540": "drilling platform, offshore rig",
+      "541": "drum, membranophone, tympan",
+      "542": "drumstick",
+      "543": "dumbbell",
+      "544": "Dutch oven",
+      "545": "electric fan, blower",
+      "546": "electric guitar",
+      "547": "electric locomotive",
+      "548": "entertainment center",
+      "549": "envelope",
+      "550": "espresso maker",
+      "551": "face powder",
+      "552": "feather boa, boa",
+      "553": "file, file cabinet, filing cabinet",
+      "554": "fireboat",
+      "555": "fire engine, fire truck",
+      "556": "fire screen, fireguard",
+      "557": "flagpole, flagstaff",
+      "558": "flute, transverse flute",
+      "559": "folding chair",
+      "560": "football helmet",
+      "561": "forklift",
+      "562": "fountain",
+      "563": "fountain pen",
+      "564": "four-poster",
+      "565": "freight car",
+      "566": "French horn, horn",
+      "567": "frying pan, frypan, skillet",
+      "568": "fur coat",
+      "569": "garbage truck, dustcart",
+      "570": "gasmask, respirator, gas helmet",
+      "571": "gas pump, gasoline pump, petrol pump, island dispenser",
+      "572": "goblet",
+      "573": "go-kart",
+      "574": "golf ball",
+      "575": "golfcart, golf cart",
+      "576": "gondola",
+      "577": "gong, tam-tam",
+      "578": "gown",
+      "579": "grand piano, grand",
+      "580": "greenhouse, nursery, glasshouse",
+      "581": "grille, radiator grille",
+      "582": "grocery store, grocery, food market, market",
+      "583": "guillotine",
+      "584": "hair slide",
+      "585": "hair spray",
+      "586": "half track",
+      "587": "hammer",
+      "588": "hamper",
+      "589": "hand blower, blow dryer, blow drier, hair dryer, hair drier",
+      "590": "hand-held computer, hand-held microcomputer",
+      "591": "handkerchief, hankie, hanky, hankey",
+      "592": "hard disc, hard disk, fixed disk",
+      "593": "harmonica, mouth organ, harp, mouth harp",
+      "594": "harp",
+      "595": "harvester, reaper",
+      "596": "hatchet",
+      "597": "holster",
+      "598": "home theater, home theatre",
+      "599": "honeycomb",
+      "600": "hook, claw",
+      "601": "hoopskirt, crinoline",
+      "602": "horizontal bar, high bar",
+      "603": "horse cart, horse-cart",
+      "604": "hourglass",
+      "605": "iPod",
+      "606": "iron, smoothing iron",
+      "607": "jack-o'-lantern",
+      "608": "jean, blue jean, denim",
+      "609": "jeep, landrover",
+      "610": "jersey, T-shirt, tee shirt",
+      "611": "jigsaw puzzle",
+      "612": "jinrikisha, ricksha, rickshaw",
+      "613": "joystick",
+      "614": "kimono",
+      "615": "knee pad",
+      "616": "knot",
+      "617": "lab coat, laboratory coat",
+      "618": "ladle",
+      "619": "lampshade, lamp shade",
+      "620": "laptop, laptop computer",
+      "621": "lawn mower, mower",
+      "622": "lens cap, lens cover",
+      "623": "letter opener, paper knife, paperknife",
+      "624": "library",
+      "625": "lifeboat",
+      "626": "lighter, light, igniter, ignitor",
+      "627": "limousine, limo",
+      "628": "liner, ocean liner",
+      "629": "lipstick, lip rouge",
+      "630": "Loafer",
+      "631": "lotion",
+      "632": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system",
+      "633": "loupe, jeweler's loupe",
+      "634": "lumbermill, sawmill",
+      "635": "magnetic compass",
+      "636": "mailbag, postbag",
+      "637": "mailbox, letter box",
+      "638": "maillot",
+      "639": "maillot, tank suit",
+      "640": "manhole cover",
+      "641": "maraca",
+      "642": "marimba, xylophone",
+      "643": "mask",
+      "644": "matchstick",
+      "645": "maypole",
+      "646": "maze, labyrinth",
+      "647": "measuring cup",
+      "648": "medicine chest, medicine cabinet",
+      "649": "megalith, megalithic structure",
+      "650": "microphone, mike",
+      "651": "microwave, microwave oven",
+      "652": "military uniform",
+      "653": "milk can",
+      "654": "minibus",
+      "655": "miniskirt, mini",
+      "656": "minivan",
+      "657": "missile",
+      "658": "mitten",
+      "659": "mixing bowl",
+      "660": "mobile home, manufactured home",
+      "661": "Model T",
+      "662": "modem",
+      "663": "monastery",
+      "664": "monitor",
+      "665": "moped",
+      "666": "mortar",
+      "667": "mortarboard",
+      "668": "mosque",
+      "669": "mosquito net",
+      "670": "motor scooter, scooter",
+      "671": "mountain bike, all-terrain bike, off-roader",
+      "672": "mountain tent",
+      "673": "mouse, computer mouse",
+      "674": "mousetrap",
+      "675": "moving van",
+      "676": "muzzle",
+      "677": "nail",
+      "678": "neck brace",
+      "679": "necklace",
+      "680": "nipple",
+      "681": "notebook, notebook computer",
+      "682": "obelisk",
+      "683": "oboe, hautboy, hautbois",
+      "684": "ocarina, sweet potato",
+      "685": "odometer, hodometer, mileometer, milometer",
+      "686": "oil filter",
+      "687": "organ, pipe organ",
+      "688": "oscilloscope, scope, cathode-ray oscilloscope, CRO",
+      "689": "overskirt",
+      "690": "oxcart",
+      "691": "oxygen mask",
+      "692": "packet",
+      "693": "paddle, boat paddle",
+      "694": "paddlewheel, paddle wheel",
+      "695": "padlock",
+      "696": "paintbrush",
+      "697": "pajama, pyjama, pj's, jammies",
+      "698": "palace",
+      "699": "panpipe, pandean pipe, syrinx",
+      "700": "paper towel",
+      "701": "parachute, chute",
+      "702": "parallel bars, bars",
+      "703": "park bench",
+      "704": "parking meter",
+      "705": "passenger car, coach, carriage",
+      "706": "patio, terrace",
+      "707": "pay-phone, pay-station",
+      "708": "pedestal, plinth, footstall",
+      "709": "pencil box, pencil case",
+      "710": "pencil sharpener",
+      "711": "perfume, essence",
+      "712": "Petri dish",
+      "713": "photocopier",
+      "714": "pick, plectrum, plectron",
+      "715": "pickelhaube",
+      "716": "picket fence, paling",
+      "717": "pickup, pickup truck",
+      "718": "pier",
+      "719": "piggy bank, penny bank",
+      "720": "pill bottle",
+      "721": "pillow",
+      "722": "ping-pong ball",
+      "723": "pinwheel",
+      "724": "pirate, pirate ship",
+      "725": "pitcher, ewer",
+      "726": "plane, carpenter's plane, woodworking plane",
+      "727": "planetarium",
+      "728": "plastic bag",
+      "729": "plate rack",
+      "730": "plow, plough",
+      "731": "plunger, plumber's helper",
+      "732": "Polaroid camera, Polaroid Land camera",
+      "733": "pole",
+      "734": "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria",
+      "735": "poncho",
+      "736": "pool table, billiard table, snooker table",
+      "737": "pop bottle, soda bottle",
+      "738": "pot, flowerpot",
+      "739": "potter's wheel",
+      "740": "power drill",
+      "741": "prayer rug, prayer mat",
+      "742": "printer",
+      "743": "prison, prison house",
+      "744": "projectile, missile",
+      "745": "projector",
+      "746": "puck, hockey puck",
+      "747": "punching bag, punch bag, punching ball, punchball",
+      "748": "purse",
+      "749": "quill, quill pen",
+      "750": "quilt, comforter, comfort, puff",
+      "751": "racer, race car, racing car",
+      "752": "racket, racquet",
+      "753": "radiator",
+      "754": "radio, wireless",
+      "755": "radio telescope, radio reflector",
+      "756": "rain barrel",
+      "757": "recreational vehicle, RV, R.V.",
+      "758": "reel",
+      "759": "reflex camera",
+      "760": "refrigerator, icebox",
+      "761": "remote control, remote",
+      "762": "restaurant, eating house, eating place, eatery",
+      "763": "revolver, six-gun, six-shooter",
+      "764": "rifle",
+      "765": "rocking chair, rocker",
+      "766": "rotisserie",
+      "767": "rubber eraser, rubber, pencil eraser",
+      "768": "rugby ball",
+      "769": "rule, ruler",
+      "770": "running shoe",
+      "771": "safe",
+      "772": "safety pin",
+      "773": "saltshaker, salt shaker",
+      "774": "sandal",
+      "775": "sarong",
+      "776": "sax, saxophone",
+      "777": "scabbard",
+      "778": "scale, weighing machine",
+      "779": "school bus",
+      "780": "schooner",
+      "781": "scoreboard",
+      "782": "screen, CRT screen",
+      "783": "screw",
+      "784": "screwdriver",
+      "785": "seat belt, seatbelt",
+      "786": "sewing machine",
+      "787": "shield, buckler",
+      "788": "shoe shop, shoe-shop, shoe store",
+      "789": "shoji",
+      "790": "shopping basket",
+      "791": "shopping cart",
+      "792": "shovel",
+      "793": "shower cap",
+      "794": "shower curtain",
+      "795": "ski",
+      "796": "ski mask",
+      "797": "sleeping bag",
+      "798": "slide rule, slipstick",
+      "799": "sliding door",
+      "800": "slot, one-armed bandit",
+      "801": "snorkel",
+      "802": "snowmobile",
+      "803": "snowplow, snowplough",
+      "804": "soap dispenser",
+      "805": "soccer ball",
+      "806": "sock",
+      "807": "solar dish, solar collector, solar furnace",
+      "808": "sombrero",
+      "809": "soup bowl",
+      "810": "space bar",
+      "811": "space heater",
+      "812": "space shuttle",
+      "813": "spatula",
+      "814": "speedboat",
+      "815": "spider web, spider's web",
+      "816": "spindle",
+      "817": "sports car, sport car",
+      "818": "spotlight, spot",
+      "819": "stage",
+      "820": "steam locomotive",
+      "821": "steel arch bridge",
+      "822": "steel drum",
+      "823": "stethoscope",
+      "824": "stole",
+      "825": "stone wall",
+      "826": "stopwatch, stop watch",
+      "827": "stove",
+      "828": "strainer",
+      "829": "streetcar, tram, tramcar, trolley, trolley car",
+      "830": "stretcher",
+      "831": "studio couch, day bed",
+      "832": "stupa, tope",
+      "833": "submarine, pigboat, sub, U-boat",
+      "834": "suit, suit of clothes",
+      "835": "sundial",
+      "836": "sunglass",
+      "837": "sunglasses, dark glasses, shades",
+      "838": "sunscreen, sunblock, sun blocker",
+      "839": "suspension bridge",
+      "840": "swab, swob, mop",
+      "841": "sweatshirt",
+      "842": "swimming trunks, bathing trunks",
+      "843": "swing",
+      "844": "switch, electric switch, electrical switch",
+      "845": "syringe",
+      "846": "table lamp",
+      "847": "tank, army tank, armored combat vehicle, armoured combat vehicle",
+      "848": "tape player",
+      "849": "teapot",
+      "850": "teddy, teddy bear",
+      "851": "television, television system",
+      "852": "tennis ball",
+      "853": "thatch, thatched roof",
+      "854": "theater curtain, theatre curtain",
+      "855": "thimble",
+      "856": "thresher, thrasher, threshing machine",
+      "857": "throne",
+      "858": "tile roof",
+      "859": "toaster",
+      "860": "tobacco shop, tobacconist shop, tobacconist",
+      "861": "toilet seat",
+      "862": "torch",
+      "863": "totem pole",
+      "864": "tow truck, tow car, wrecker",
+      "865": "toyshop",
+      "866": "tractor",
+      "867": "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi",
+      "868": "tray",
+      "869": "trench coat",
+      "870": "tricycle, trike, velocipede",
+      "871": "trimaran",
+      "872": "tripod",
+      "873": "triumphal arch",
+      "874": "trolleybus, trolley coach, trackless trolley",
+      "875": "trombone",
+      "876": "tub, vat",
+      "877": "turnstile",
+      "878": "typewriter keyboard",
+      "879": "umbrella",
+      "880": "unicycle, monocycle",
+      "881": "upright, upright piano",
+      "882": "vacuum, vacuum cleaner",
+      "883": "vase",
+      "884": "vault",
+      "885": "velvet",
+      "886": "vending machine",
+      "887": "vestment",
+      "888": "viaduct",
+      "889": "violin, fiddle",
+      "890": "volleyball",
+      "891": "waffle iron",
+      "892": "wall clock",
+      "893": "wallet, billfold, notecase, pocketbook",
+      "894": "wardrobe, closet, press",
+      "895": "warplane, military plane",
+      "896": "washbasin, handbasin, washbowl, lavabo, wash-hand basin",
+      "897": "washer, automatic washer, washing machine",
+      "898": "water bottle",
+      "899": "water jug",
+      "900": "water tower",
+      "901": "whiskey jug",
+      "902": "whistle",
+      "903": "wig",
+      "904": "window screen",
+      "905": "window shade",
+      "906": "Windsor tie",
+      "907": "wine bottle",
+      "908": "wing",
+      "909": "wok",
+      "910": "wooden spoon",
+      "911": "wool, woolen, woollen",
+      "912": "worm fence, snake fence, snake-rail fence, Virginia fence",
+      "913": "wreck",
+      "914": "yawl",
+      "915": "yurt",
+      "916": "web site, website, internet site, site",
+      "917": "comic book",
+      "918": "crossword puzzle, crossword",
+      "919": "street sign",
+      "920": "traffic light, traffic signal, stoplight",
+      "921": "book jacket, dust cover, dust jacket, dust wrapper",
+      "922": "menu",
+      "923": "plate",
+      "924": "guacamole",
+      "925": "consomme",
+      "926": "hot pot, hotpot",
+      "927": "trifle",
+      "928": "ice cream, icecream",
+      "929": "ice lolly, lolly, lollipop, popsicle",
+      "930": "French loaf",
+      "931": "bagel, beigel",
+      "932": "pretzel",
+      "933": "cheeseburger",
+      "934": "hotdog, hot dog, red hot",
+      "935": "mashed potato",
+      "936": "head cabbage",
+      "937": "broccoli",
+      "938": "cauliflower",
+      "939": "zucchini, courgette",
+      "940": "spaghetti squash",
+      "941": "acorn squash",
+      "942": "butternut squash",
+      "943": "cucumber, cuke",
+      "944": "artichoke, globe artichoke",
+      "945": "bell pepper",
+      "946": "cardoon",
+      "947": "mushroom",
+      "948": "Granny Smith",
+      "949": "strawberry",
+      "950": "orange",
+      "951": "lemon",
+      "952": "fig",
+      "953": "pineapple, ananas",
+      "954": "banana",
+      "955": "jackfruit, jak, jack",
+      "956": "custard apple",
+      "957": "pomegranate",
+      "958": "hay",
+      "959": "carbonara",
+      "960": "chocolate sauce, chocolate syrup",
+      "961": "dough",
+      "962": "meat loaf, meatloaf",
+      "963": "pizza, pizza pie",
+      "964": "potpie",
+      "965": "burrito",
+      "966": "red wine",
+      "967": "espresso",
+      "968": "cup",
+      "969": "eggnog",
+      "970": "alp",
+      "971": "bubble",
+      "972": "cliff, drop, drop-off",
+      "973": "coral reef",
+      "974": "geyser",
+      "975": "lakeside, lakeshore",
+      "976": "promontory, headland, head, foreland",
+      "977": "sandbar, sand bar",
+      "978": "seashore, coast, seacoast, sea-coast",
+      "979": "valley, vale",
+      "980": "volcano",
+      "981": "ballplayer, baseball player",
+      "982": "groom, bridegroom",
+      "983": "scuba diver",
+      "984": "rapeseed",
+      "985": "daisy",
+      "986": "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum",
+      "987": "corn",
+      "988": "acorn",
+      "989": "hip, rose hip, rosehip",
+      "990": "buckeye, horse chestnut, conker",
+      "991": "coral fungus",
+      "992": "agaric",
+      "993": "gyromitra",
+      "994": "stinkhorn, carrion fungus",
+      "995": "earthstar",
+      "996": "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa",
+      "997": "bolete",
+      "998": "ear, spike, capitulum",
+      "999": "toilet tissue, toilet paper, bathroom tissue"
+    },
+    "image_size": 256,
+    "label2id": {
+      "Afghan hound, Afghan": 160,
+      "African chameleon, Chamaeleo chamaeleon": 47,
+      "African crocodile, Nile crocodile, Crocodylus niloticus": 49,
+      "African elephant, Loxodonta africana": 386,
+      "African grey, African gray, Psittacus erithacus": 87,
+      "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus": 275,
+      "Airedale, Airedale terrier": 191,
+      "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier": 180,
+      "American alligator, Alligator mississipiensis": 50,
+      "American black bear, black bear, Ursus americanus, Euarctos americanus": 295,
+      "American chameleon, anole, Anolis carolinensis": 40,
+      "American coot, marsh hen, mud hen, water hen, Fulica americana": 137,
+      "American egret, great white heron, Egretta albus": 132,
+      "American lobster, Northern lobster, Maine lobster, Homarus americanus": 122,
+      "Angora, Angora rabbit": 332,
+      "Appenzeller": 240,
+      "Arabian camel, dromedary, Camelus dromedarius": 354,
+      "Arctic fox, white fox, Alopex lagopus": 279,
+      "Australian terrier": 193,
+      "Band Aid": 419,
+      "Bedlington terrier": 181,
+      "Bernese mountain dog": 239,
+      "Blenheim spaniel": 156,
+      "Border collie": 232,
+      "Border terrier": 182,
+      "Boston bull, Boston terrier": 195,
+      "Bouvier des Flandres, Bouviers des Flandres": 233,
+      "Brabancon griffon": 262,
+      "Brittany spaniel": 215,
+      "CD player": 485,
+      "Cardigan, Cardigan Welsh corgi": 264,
+      "Chesapeake Bay retriever": 209,
+      "Chihuahua": 151,
+      "Christmas stocking": 496,
+      "Crock Pot": 521,
+      "Dandie Dinmont, Dandie Dinmont terrier": 194,
+      "Doberman, Doberman pinscher": 236,
+      "Dungeness crab, Cancer magister": 118,
+      "Dutch oven": 544,
+      "Egyptian cat": 285,
+      "English foxhound": 167,
+      "English setter": 212,
+      "English springer, English springer spaniel": 217,
+      "EntleBucher": 241,
+      "Eskimo dog, husky": 248,
+      "European fire salamander, Salamandra salamandra": 25,
+      "European gallinule, Porphyrio porphyrio": 136,
+      "French bulldog": 245,
+      "French horn, horn": 566,
+      "French loaf": 930,
+      "German shepherd, German shepherd dog, German police dog, alsatian": 235,
+      "German short-haired pointer": 210,
+      "Gila monster, Heloderma suspectum": 45,
+      "Gordon setter": 214,
+      "Granny Smith": 948,
+      "Great Dane": 246,
+      "Great Pyrenees": 257,
+      "Greater Swiss Mountain dog": 238,
+      "Ibizan hound, Ibizan Podenco": 173,
+      "Indian cobra, Naja naja": 63,
+      "Indian elephant, Elephas maximus": 385,
+      "Irish setter, red setter": 213,
+      "Irish terrier": 184,
+      "Irish water spaniel": 221,
+      "Irish wolfhound": 170,
+      "Italian greyhound": 171,
+      "Japanese spaniel": 152,
+      "Kerry blue terrier": 183,
+      "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis": 48,
+      "Labrador retriever": 208,
+      "Lakeland terrier": 189,
+      "Leonberg": 255,
+      "Lhasa, Lhasa apso": 204,
+      "Loafer": 630,
+      "Madagascar cat, ring-tailed lemur, Lemur catta": 383,
+      "Maltese dog, Maltese terrier, Maltese": 153,
+      "Mexican hairless": 268,
+      "Model T": 661,
+      "Newfoundland, Newfoundland dog": 256,
+      "Norfolk terrier": 185,
+      "Norwegian elkhound, elkhound": 174,
+      "Norwich terrier": 186,
+      "Old English sheepdog, bobtail": 229,
+      "Pekinese, Pekingese, Peke": 154,
+      "Pembroke, Pembroke Welsh corgi": 263,
+      "Persian cat": 283,
+      "Petri dish": 712,
+      "Polaroid camera, Polaroid Land camera": 732,
+      "Pomeranian": 259,
+      "Rhodesian ridgeback": 159,
+      "Rottweiler": 234,
+      "Saint Bernard, St Bernard": 247,
+      "Saluki, gazelle hound": 176,
+      "Samoyed, Samoyede": 258,
+      "Scotch terrier, Scottish terrier, Scottie": 199,
+      "Scottish deerhound, deerhound": 177,
+      "Sealyham terrier, Sealyham": 190,
+      "Shetland sheepdog, Shetland sheep dog, Shetland": 230,
+      "Shih-Tzu": 155,
+      "Siamese cat, Siamese": 284,
+      "Siberian husky": 250,
+      "Staffordshire bullterrier, Staffordshire bull terrier": 179,
+      "Sussex spaniel": 220,
+      "Tibetan mastiff": 244,
+      "Tibetan terrier, chrysanthemum dog": 200,
+      "Walker hound, Walker foxhound": 166,
+      "Weimaraner": 178,
+      "Welsh springer spaniel": 218,
+      "West Highland white terrier": 203,
+      "Windsor tie": 906,
+      "Yorkshire terrier": 187,
+      "abacus": 398,
+      "abaya": 399,
+      "academic gown, academic robe, judge's robe": 400,
+      "accordion, piano accordion, squeeze box": 401,
+      "acorn": 988,
+      "acorn squash": 941,
+      "acoustic guitar": 402,
+      "admiral": 321,
+      "affenpinscher, monkey pinscher, monkey dog": 252,
+      "agama": 42,
+      "agaric": 992,
+      "aircraft carrier, carrier, flattop, attack aircraft carrier": 403,
+      "airliner": 404,
+      "airship, dirigible": 405,
+      "albatross, mollymawk": 146,
+      "alligator lizard": 44,
+      "alp": 970,
+      "altar": 406,
+      "ambulance": 407,
+      "amphibian, amphibious vehicle": 408,
+      "analog clock": 409,
+      "anemone fish": 393,
+      "ant, emmet, pismire": 310,
+      "apiary, bee house": 410,
+      "apron": 411,
+      "armadillo": 363,
+      "artichoke, globe artichoke": 944,
+      "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin": 412,
+      "assault rifle, assault gun": 413,
+      "axolotl, mud puppy, Ambystoma mexicanum": 29,
+      "baboon": 372,
+      "backpack, back pack, knapsack, packsack, rucksack, haversack": 414,
+      "badger": 362,
+      "bagel, beigel": 931,
+      "bakery, bakeshop, bakehouse": 415,
+      "balance beam, beam": 416,
+      "bald eagle, American eagle, Haliaeetus leucocephalus": 22,
+      "balloon": 417,
+      "ballplayer, baseball player": 981,
+      "ballpoint, ballpoint pen, ballpen, Biro": 418,
+      "banana": 954,
+      "banded gecko": 38,
+      "banjo": 420,
+      "bannister, banister, balustrade, balusters, handrail": 421,
+      "barbell": 422,
+      "barber chair": 423,
+      "barbershop": 424,
+      "barn": 425,
+      "barn spider, Araneus cavaticus": 73,
+      "barometer": 426,
+      "barracouta, snoek": 389,
+      "barrel, cask": 427,
+      "barrow, garden cart, lawn cart, wheelbarrow": 428,
+      "baseball": 429,
+      "basenji": 253,
+      "basketball": 430,
+      "basset, basset hound": 161,
+      "bassinet": 431,
+      "bassoon": 432,
+      "bath towel": 434,
+      "bathing cap, swimming cap": 433,
+      "bathtub, bathing tub, bath, tub": 435,
+      "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon": 436,
+      "beacon, lighthouse, beacon light, pharos": 437,
+      "beagle": 162,
+      "beaker": 438,
+      "bearskin, busby, shako": 439,
+      "beaver": 337,
+      "bee": 309,
+      "bee eater": 92,
+      "beer bottle": 440,
+      "beer glass": 441,
+      "bell cote, bell cot": 442,
+      "bell pepper": 945,
+      "bib": 443,
+      "bicycle-built-for-two, tandem bicycle, tandem": 444,
+      "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis": 349,
+      "bikini, two-piece": 445,
+      "binder, ring-binder": 446,
+      "binoculars, field glasses, opera glasses": 447,
+      "birdhouse": 448,
+      "bison": 347,
+      "bittern": 133,
+      "black and gold garden spider, Argiope aurantia": 72,
+      "black grouse": 80,
+      "black stork, Ciconia nigra": 128,
+      "black swan, Cygnus atratus": 100,
+      "black widow, Latrodectus mactans": 75,
+      "black-and-tan coonhound": 165,
+      "black-footed ferret, ferret, Mustela nigripes": 359,
+      "bloodhound, sleuthhound": 163,
+      "bluetick": 164,
+      "boa constrictor, Constrictor constrictor": 61,
+      "boathouse": 449,
+      "bobsled, bobsleigh, bob": 450,
+      "bolete": 997,
+      "bolo tie, bolo, bola tie, bola": 451,
+      "bonnet, poke bonnet": 452,
+      "book jacket, dust cover, dust jacket, dust wrapper": 921,
+      "bookcase": 453,
+      "bookshop, bookstore, bookstall": 454,
+      "borzoi, Russian wolfhound": 169,
+      "bottlecap": 455,
+      "bow": 456,
+      "bow tie, bow-tie, bowtie": 457,
+      "box turtle, box tortoise": 37,
+      "boxer": 242,
+      "brain coral": 109,
+      "brambling, Fringilla montifringilla": 10,
+      "brass, memorial tablet, plaque": 458,
+      "brassiere, bra, bandeau": 459,
+      "breakwater, groin, groyne, mole, bulwark, seawall, jetty": 460,
+      "breastplate, aegis, egis": 461,
+      "briard": 226,
+      "broccoli": 937,
+      "broom": 462,
+      "brown bear, bruin, Ursus arctos": 294,
+      "bubble": 971,
+      "bucket, pail": 463,
+      "buckeye, horse chestnut, conker": 990,
+      "buckle": 464,
+      "bulbul": 16,
+      "bull mastiff": 243,
+      "bullet train, bullet": 466,
+      "bulletproof vest": 465,
+      "bullfrog, Rana catesbeiana": 30,
+      "burrito": 965,
+      "bustard": 138,
+      "butcher shop, meat market": 467,
+      "butternut squash": 942,
+      "cab, hack, taxi, taxicab": 468,
+      "cabbage butterfly": 324,
+      "cairn, cairn terrier": 192,
+      "caldron, cauldron": 469,
+      "can opener, tin opener": 473,
+      "candle, taper, wax light": 470,
+      "cannon": 471,
+      "canoe": 472,
+      "capuchin, ringtail, Cebus capucinus": 378,
+      "car mirror": 475,
+      "car wheel": 479,
+      "carbonara": 959,
+      "cardigan": 474,
+      "cardoon": 946,
+      "carousel, carrousel, merry-go-round, roundabout, whirligig": 476,
+      "carpenter's kit, tool kit": 477,
+      "carton": 478,
+      "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM": 480,
+      "cassette": 481,
+      "cassette player": 482,
+      "castle": 483,
+      "catamaran": 484,
+      "cauliflower": 938,
+      "cello, violoncello": 486,
+      "cellular telephone, cellular phone, cellphone, cell, mobile phone": 487,
+      "centipede": 79,
+      "chain": 488,
+      "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour": 490,
+      "chain saw, chainsaw": 491,
+      "chainlink fence": 489,
+      "chambered nautilus, pearly nautilus, nautilus": 117,
+      "cheeseburger": 933,
+      "cheetah, chetah, Acinonyx jubatus": 293,
+      "chest": 492,
+      "chickadee": 19,
+      "chiffonier, commode": 493,
+      "chime, bell, gong": 494,
+      "chimpanzee, chimp, Pan troglodytes": 367,
+      "china cabinet, china closet": 495,
+      "chiton, coat-of-mail shell, sea cradle, polyplacophore": 116,
+      "chocolate sauce, chocolate syrup": 960,
+      "chow, chow chow": 260,
+      "church, church building": 497,
+      "cicada, cicala": 316,
+      "cinema, movie theater, movie theatre, movie house, picture palace": 498,
+      "cleaver, meat cleaver, chopper": 499,
+      "cliff dwelling": 500,
+      "cliff, drop, drop-off": 972,
+      "cloak": 501,
+      "clog, geta, patten, sabot": 502,
+      "clumber, clumber spaniel": 216,
+      "cock": 7,
+      "cocker spaniel, English cocker spaniel, cocker": 219,
+      "cockroach, roach": 314,
+      "cocktail shaker": 503,
+      "coffee mug": 504,
+      "coffeepot": 505,
+      "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch": 391,
+      "coil, spiral, volute, whorl, helix": 506,
+      "collie": 231,
+      "colobus, colobus monkey": 375,
+      "combination lock": 507,
+      "comic book": 917,
+      "common iguana, iguana, Iguana iguana": 39,
+      "common newt, Triturus vulgaris": 26,
+      "computer keyboard, keypad": 508,
+      "conch": 112,
+      "confectionery, confectionary, candy store": 509,
+      "consomme": 925,
+      "container ship, containership, container vessel": 510,
+      "convertible": 511,
+      "coral fungus": 991,
+      "coral reef": 973,
+      "corkscrew, bottle screw": 512,
+      "corn": 987,
+      "cornet, horn, trumpet, trump": 513,
+      "coucal": 91,
+      "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor": 286,
+      "cowboy boot": 514,
+      "cowboy hat, ten-gallon hat": 515,
+      "coyote, prairie wolf, brush wolf, Canis latrans": 272,
+      "cradle": 516,
+      "crane": 517,
+      "crash helmet": 518,
+      "crate": 519,
+      "crayfish, crawfish, crawdad, crawdaddy": 124,
+      "crib, cot": 520,
+      "cricket": 312,
+      "croquet ball": 522,
+      "crossword puzzle, crossword": 918,
+      "crutch": 523,
+      "cucumber, cuke": 943,
+      "cuirass": 524,
+      "cup": 968,
+      "curly-coated retriever": 206,
+      "custard apple": 956,
+      "daisy": 985,
+      "dalmatian, coach dog, carriage dog": 251,
+      "dam, dike, dyke": 525,
+      "damselfly": 320,
+      "desk": 526,
+      "desktop computer": 527,
+      "dhole, Cuon alpinus": 274,
+      "dial telephone, dial phone": 528,
+      "diamondback, diamondback rattlesnake, Crotalus adamanteus": 67,
+      "diaper, nappy, napkin": 529,
+      "digital clock": 530,
+      "digital watch": 531,
+      "dingo, warrigal, warragal, Canis dingo": 273,
+      "dining table, board": 532,
+      "dishrag, dishcloth": 533,
+      "dishwasher, dish washer, dishwashing machine": 534,
+      "disk brake, disc brake": 535,
+      "dock, dockage, docking facility": 536,
+      "dogsled, dog sled, dog sleigh": 537,
+      "dome": 538,
+      "doormat, welcome mat": 539,
+      "dough": 961,
+      "dowitcher": 142,
+      "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk": 319,
+      "drake": 97,
+      "drilling platform, offshore rig": 540,
+      "drum, membranophone, tympan": 541,
+      "drumstick": 542,
+      "dugong, Dugong dugon": 149,
+      "dumbbell": 543,
+      "dung beetle": 305,
+      "ear, spike, capitulum": 998,
+      "earthstar": 995,
+      "echidna, spiny anteater, anteater": 102,
+      "eel": 390,
+      "eft": 27,
+      "eggnog": 969,
+      "electric fan, blower": 545,
+      "electric guitar": 546,
+      "electric locomotive": 547,
+      "electric ray, crampfish, numbfish, torpedo": 5,
+      "entertainment center": 548,
+      "envelope": 549,
+      "espresso": 967,
+      "espresso maker": 550,
+      "face powder": 551,
+      "feather boa, boa": 552,
+      "fiddler crab": 120,
+      "fig": 952,
+      "file, file cabinet, filing cabinet": 553,
+      "fire engine, fire truck": 555,
+      "fire screen, fireguard": 556,
+      "fireboat": 554,
+      "flagpole, flagstaff": 557,
+      "flamingo": 130,
+      "flat-coated retriever": 205,
+      "flatworm, platyhelminth": 110,
+      "flute, transverse flute": 558,
+      "fly": 308,
+      "folding chair": 559,
+      "football helmet": 560,
+      "forklift": 561,
+      "fountain": 562,
+      "fountain pen": 563,
+      "four-poster": 564,
+      "fox squirrel, eastern fox squirrel, Sciurus niger": 335,
+      "freight car": 565,
+      "frilled lizard, Chlamydosaurus kingi": 43,
+      "frying pan, frypan, skillet": 567,
+      "fur coat": 568,
+      "gar, garfish, garpike, billfish, Lepisosteus osseus": 395,
+      "garbage truck, dustcart": 569,
+      "garden spider, Aranea diademata": 74,
+      "garter snake, grass snake": 57,
+      "gas pump, gasoline pump, petrol pump, island dispenser": 571,
+      "gasmask, respirator, gas helmet": 570,
+      "gazelle": 353,
+      "geyser": 974,
+      "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca": 388,
+      "giant schnauzer": 197,
+      "gibbon, Hylobates lar": 368,
+      "go-kart": 573,
+      "goblet": 572,
+      "golden retriever": 207,
+      "goldfinch, Carduelis carduelis": 11,
+      "goldfish, Carassius auratus": 1,
+      "golf ball": 574,
+      "golfcart, golf cart": 575,
+      "gondola": 576,
+      "gong, tam-tam": 577,
+      "goose": 99,
+      "gorilla, Gorilla gorilla": 366,
+      "gown": 578,
+      "grand piano, grand": 579,
+      "grasshopper, hopper": 311,
+      "great grey owl, great gray owl, Strix nebulosa": 24,
+      "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias": 2,
+      "green lizard, Lacerta viridis": 46,
+      "green mamba": 64,
+      "green snake, grass snake": 55,
+      "greenhouse, nursery, glasshouse": 580,
+      "grey fox, gray fox, Urocyon cinereoargenteus": 280,
+      "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus": 147,
+      "grille, radiator grille": 581,
+      "grocery store, grocery, food market, market": 582,
+      "groenendael": 224,
+      "groom, bridegroom": 982,
+      "ground beetle, carabid beetle": 302,
+      "guacamole": 924,
+      "guenon, guenon monkey": 370,
+      "guillotine": 583,
+      "guinea pig, Cavia cobaya": 338,
+      "gyromitra": 993,
+      "hair slide": 584,
+      "hair spray": 585,
+      "half track": 586,
+      "hammer": 587,
+      "hammerhead, hammerhead shark": 4,
+      "hamper": 588,
+      "hamster": 333,
+      "hand blower, blow dryer, blow drier, hair dryer, hair drier": 589,
+      "hand-held computer, hand-held microcomputer": 590,
+      "handkerchief, hankie, hanky, hankey": 591,
+      "hard disc, hard disk, fixed disk": 592,
+      "hare": 331,
+      "harmonica, mouth organ, harp, mouth harp": 593,
+      "harp": 594,
+      "hartebeest": 351,
+      "harvester, reaper": 595,
+      "harvestman, daddy longlegs, Phalangium opilio": 70,
+      "hatchet": 596,
+      "hay": 958,
+      "head cabbage": 936,
+      "hen": 8,
+      "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa": 996,
+      "hermit crab": 125,
+      "hip, rose hip, rosehip": 989,
+      "hippopotamus, hippo, river horse, Hippopotamus amphibius": 344,
+      "hog, pig, grunter, squealer, Sus scrofa": 341,
+      "hognose snake, puff adder, sand viper": 54,
+      "holster": 597,
+      "home theater, home theatre": 598,
+      "honeycomb": 599,
+      "hook, claw": 600,
+      "hoopskirt, crinoline": 601,
+      "horizontal bar, high bar": 602,
+      "hornbill": 93,
+      "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus": 66,
+      "horse cart, horse-cart": 603,
+      "hot pot, hotpot": 926,
+      "hotdog, hot dog, red hot": 934,
+      "hourglass": 604,
+      "house finch, linnet, Carpodacus mexicanus": 12,
+      "howler monkey, howler": 379,
+      "hummingbird": 94,
+      "hyena, hyaena": 276,
+      "iPod": 605,
+      "ibex, Capra ibex": 350,
+      "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus": 296,
+      "ice cream, icecream": 928,
+      "ice lolly, lolly, lollipop, popsicle": 929,
+      "impala, Aepyceros melampus": 352,
+      "indigo bunting, indigo finch, indigo bird, Passerina cyanea": 14,
+      "indri, indris, Indri indri, Indri brevicaudatus": 384,
+      "iron, smoothing iron": 606,
+      "isopod": 126,
+      "jacamar": 95,
+      "jack-o'-lantern": 607,
+      "jackfruit, jak, jack": 955,
+      "jaguar, panther, Panthera onca, Felis onca": 290,
+      "jay": 17,
+      "jean, blue jean, denim": 608,
+      "jeep, landrover": 609,
+      "jellyfish": 107,
+      "jersey, T-shirt, tee shirt": 610,
+      "jigsaw puzzle": 611,
+      "jinrikisha, ricksha, rickshaw": 612,
+      "joystick": 613,
+      "junco, snowbird": 13,
+      "keeshond": 261,
+      "kelpie": 227,
+      "killer whale, killer, orca, grampus, sea wolf, Orcinus orca": 148,
+      "kimono": 614,
+      "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica": 121,
+      "king penguin, Aptenodytes patagonica": 145,
+      "king snake, kingsnake": 56,
+      "kit fox, Vulpes macrotis": 278,
+      "kite": 21,
+      "knee pad": 615,
+      "knot": 616,
+      "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus": 105,
+      "komondor": 228,
+      "kuvasz": 222,
+      "lab coat, laboratory coat": 617,
+      "lacewing, lacewing fly": 318,
+      "ladle": 618,
+      "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle": 301,
+      "lakeside, lakeshore": 975,
+      "lampshade, lamp shade": 619,
+      "langur": 374,
+      "laptop, laptop computer": 620,
+      "lawn mower, mower": 621,
+      "leaf beetle, chrysomelid": 304,
+      "leafhopper": 317,
+      "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea": 34,
+      "lemon": 951,
+      "lens cap, lens cover": 622,
+      "leopard, Panthera pardus": 288,
+      "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens": 387,
+      "letter opener, paper knife, paperknife": 623,
+      "library": 624,
+      "lifeboat": 625,
+      "lighter, light, igniter, ignitor": 626,
+      "limousine, limo": 627,
+      "limpkin, Aramus pictus": 135,
+      "liner, ocean liner": 628,
+      "lion, king of beasts, Panthera leo": 291,
+      "lionfish": 396,
+      "lipstick, lip rouge": 629,
+      "little blue heron, Egretta caerulea": 131,
+      "llama": 355,
+      "loggerhead, loggerhead turtle, Caretta caretta": 33,
+      "long-horned beetle, longicorn, longicorn beetle": 303,
+      "lorikeet": 90,
+      "lotion": 631,
+      "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system": 632,
+      "loupe, jeweler's loupe": 633,
+      "lumbermill, sawmill": 634,
+      "lycaenid, lycaenid butterfly": 326,
+      "lynx, catamount": 287,
+      "macaque": 373,
+      "macaw": 88,
+      "magnetic compass": 635,
+      "magpie": 18,
+      "mailbag, postbag": 636,
+      "mailbox, letter box": 637,
+      "maillot": 638,
+      "maillot, tank suit": 639,
+      "malamute, malemute, Alaskan malamute": 249,
+      "malinois": 225,
+      "manhole cover": 640,
+      "mantis, mantid": 315,
+      "maraca": 641,
+      "marimba, xylophone": 642,
+      "marmoset": 377,
+      "marmot": 336,
+      "mashed potato": 935,
+      "mask": 643,
+      "matchstick": 644,
+      "maypole": 645,
+      "maze, labyrinth": 646,
+      "measuring cup": 647,
+      "meat loaf, meatloaf": 962,
+      "medicine chest, medicine cabinet": 648,
+      "meerkat, mierkat": 299,
+      "megalith, megalithic structure": 649,
+      "menu": 922,
+      "microphone, mike": 650,
+      "microwave, microwave oven": 651,
+      "military uniform": 652,
+      "milk can": 653,
+      "miniature pinscher": 237,
+      "miniature poodle": 266,
+      "miniature schnauzer": 196,
+      "minibus": 654,
+      "miniskirt, mini": 655,
+      "minivan": 656,
+      "mink": 357,
+      "missile": 657,
+      "mitten": 658,
+      "mixing bowl": 659,
+      "mobile home, manufactured home": 660,
+      "modem": 662,
+      "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus": 323,
+      "monastery": 663,
+      "mongoose": 298,
+      "monitor": 664,
+      "moped": 665,
+      "mortar": 666,
+      "mortarboard": 667,
+      "mosque": 668,
+      "mosquito net": 669,
+      "motor scooter, scooter": 670,
+      "mountain bike, all-terrain bike, off-roader": 671,
+      "mountain tent": 672,
+      "mouse, computer mouse": 673,
+      "mousetrap": 674,
+      "moving van": 675,
+      "mud turtle": 35,
+      "mushroom": 947,
+      "muzzle": 676,
+      "nail": 677,
+      "neck brace": 678,
+      "necklace": 679,
+      "nematode, nematode worm, roundworm": 111,
+      "night snake, Hypsiglena torquata": 60,
+      "nipple": 680,
+      "notebook, notebook computer": 681,
+      "obelisk": 682,
+      "oboe, hautboy, hautbois": 683,
+      "ocarina, sweet potato": 684,
+      "odometer, hodometer, mileometer, milometer": 685,
+      "oil filter": 686,
+      "orange": 950,
+      "orangutan, orang, orangutang, Pongo pygmaeus": 365,
+      "organ, pipe organ": 687,
+      "oscilloscope, scope, cathode-ray oscilloscope, CRO": 688,
+      "ostrich, Struthio camelus": 9,
+      "otter": 360,
+      "otterhound, otter hound": 175,
+      "overskirt": 689,
+      "ox": 345,
+      "oxcart": 690,
+      "oxygen mask": 691,
+      "oystercatcher, oyster catcher": 143,
+      "packet": 692,
+      "paddle, boat paddle": 693,
+      "paddlewheel, paddle wheel": 694,
+      "padlock": 695,
+      "paintbrush": 696,
+      "pajama, pyjama, pj's, jammies": 697,
+      "palace": 698,
+      "panpipe, pandean pipe, syrinx": 699,
+      "paper towel": 700,
+      "papillon": 157,
+      "parachute, chute": 701,
+      "parallel bars, bars": 702,
+      "park bench": 703,
+      "parking meter": 704,
+      "partridge": 86,
+      "passenger car, coach, carriage": 705,
+      "patas, hussar monkey, Erythrocebus patas": 371,
+      "patio, terrace": 706,
+      "pay-phone, pay-station": 707,
+      "peacock": 84,
+      "pedestal, plinth, footstall": 708,
+      "pelican": 144,
+      "pencil box, pencil case": 709,
+      "pencil sharpener": 710,
+      "perfume, essence": 711,
+      "photocopier": 713,
+      "pick, plectrum, plectron": 714,
+      "pickelhaube": 715,
+      "picket fence, paling": 716,
+      "pickup, pickup truck": 717,
+      "pier": 718,
+      "piggy bank, penny bank": 719,
+      "pill bottle": 720,
+      "pillow": 721,
+      "pineapple, ananas": 953,
+      "ping-pong ball": 722,
+      "pinwheel": 723,
+      "pirate, pirate ship": 724,
+      "pitcher, ewer": 725,
+      "pizza, pizza pie": 963,
+      "plane, carpenter's plane, woodworking plane": 726,
+      "planetarium": 727,
+      "plastic bag": 728,
+      "plate": 923,
+      "plate rack": 729,
+      "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus": 103,
+      "plow, plough": 730,
+      "plunger, plumber's helper": 731,
+      "pole": 733,
+      "polecat, fitch, foulmart, foumart, Mustela putorius": 358,
+      "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria": 734,
+      "pomegranate": 957,
+      "poncho": 735,
+      "pool table, billiard table, snooker table": 736,
+      "pop bottle, soda bottle": 737,
+      "porcupine, hedgehog": 334,
+      "pot, flowerpot": 738,
+      "potpie": 964,
+      "potter's wheel": 739,
+      "power drill": 740,
+      "prairie chicken, prairie grouse, prairie fowl": 83,
+      "prayer rug, prayer mat": 741,
+      "pretzel": 932,
+      "printer": 742,
+      "prison, prison house": 743,
+      "proboscis monkey, Nasalis larvatus": 376,
+      "projectile, missile": 744,
+      "projector": 745,
+      "promontory, headland, head, foreland": 976,
+      "ptarmigan": 81,
+      "puck, hockey puck": 746,
+      "puffer, pufferfish, blowfish, globefish": 397,
+      "pug, pug-dog": 254,
+      "punching bag, punch bag, punching ball, punchball": 747,
+      "purse": 748,
+      "quail": 85,
+      "quill, quill pen": 749,
+      "quilt, comforter, comfort, puff": 750,
+      "racer, race car, racing car": 751,
+      "racket, racquet": 752,
+      "radiator": 753,
+      "radio telescope, radio reflector": 755,
+      "radio, wireless": 754,
+      "rain barrel": 756,
+      "ram, tup": 348,
+      "rapeseed": 984,
+      "recreational vehicle, RV, R.V.": 757,
+      "red fox, Vulpes vulpes": 277,
+      "red wine": 966,
+      "red wolf, maned wolf, Canis rufus, Canis niger": 271,
+      "red-backed sandpiper, dunlin, Erolia alpina": 140,
+      "red-breasted merganser, Mergus serrator": 98,
+      "redbone": 168,
+      "redshank, Tringa totanus": 141,
+      "reel": 758,
+      "reflex camera": 759,
+      "refrigerator, icebox": 760,
+      "remote control, remote": 761,
+      "restaurant, eating house, eating place, eatery": 762,
+      "revolver, six-gun, six-shooter": 763,
+      "rhinoceros beetle": 306,
+      "rifle": 764,
+      "ringlet, ringlet butterfly": 322,
+      "ringneck snake, ring-necked snake, ring snake": 53,
+      "robin, American robin, Turdus migratorius": 15,
+      "rock beauty, Holocanthus tricolor": 392,
+      "rock crab, Cancer irroratus": 119,
+      "rock python, rock snake, Python sebae": 62,
+      "rocking chair, rocker": 765,
+      "rotisserie": 766,
+      "rubber eraser, rubber, pencil eraser": 767,
+      "ruddy turnstone, Arenaria interpres": 139,
+      "ruffed grouse, partridge, Bonasa umbellus": 82,
+      "rugby ball": 768,
+      "rule, ruler": 769,
+      "running shoe": 770,
+      "safe": 771,
+      "safety pin": 772,
+      "saltshaker, salt shaker": 773,
+      "sandal": 774,
+      "sandbar, sand bar": 977,
+      "sarong": 775,
+      "sax, saxophone": 776,
+      "scabbard": 777,
+      "scale, weighing machine": 778,
+      "schipperke": 223,
+      "school bus": 779,
+      "schooner": 780,
+      "scoreboard": 781,
+      "scorpion": 71,
+      "screen, CRT screen": 782,
+      "screw": 783,
+      "screwdriver": 784,
+      "scuba diver": 983,
+      "sea anemone, anemone": 108,
+      "sea cucumber, holothurian": 329,
+      "sea lion": 150,
+      "sea slug, nudibranch": 115,
+      "sea snake": 65,
+      "sea urchin": 328,
+      "seashore, coast, seacoast, sea-coast": 978,
+      "seat belt, seatbelt": 785,
+      "sewing machine": 786,
+      "shield, buckler": 787,
+      "shoe shop, shoe-shop, shoe store": 788,
+      "shoji": 789,
+      "shopping basket": 790,
+      "shopping cart": 791,
+      "shovel": 792,
+      "shower cap": 793,
+      "shower curtain": 794,
+      "siamang, Hylobates syndactylus, Symphalangus syndactylus": 369,
+      "sidewinder, horned rattlesnake, Crotalus cerastes": 68,
+      "silky terrier, Sydney silky": 201,
+      "ski": 795,
+      "ski mask": 796,
+      "skunk, polecat, wood pussy": 361,
+      "sleeping bag": 797,
+      "slide rule, slipstick": 798,
+      "sliding door": 799,
+      "slot, one-armed bandit": 800,
+      "sloth bear, Melursus ursinus, Ursus ursinus": 297,
+      "slug": 114,
+      "snail": 113,
+      "snorkel": 801,
+      "snow leopard, ounce, Panthera uncia": 289,
+      "snowmobile": 802,
+      "snowplow, snowplough": 803,
+      "soap dispenser": 804,
+      "soccer ball": 805,
+      "sock": 806,
+      "soft-coated wheaten terrier": 202,
+      "solar dish, solar collector, solar furnace": 807,
+      "sombrero": 808,
+      "sorrel": 339,
+      "soup bowl": 809,
+      "space bar": 810,
+      "space heater": 811,
+      "space shuttle": 812,
+      "spaghetti squash": 940,
+      "spatula": 813,
+      "speedboat": 814,
+      "spider monkey, Ateles geoffroyi": 381,
+      "spider web, spider's web": 815,
+      "spindle": 816,
+      "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish": 123,
+      "spoonbill": 129,
+      "sports car, sport car": 817,
+      "spotlight, spot": 818,
+      "spotted salamander, Ambystoma maculatum": 28,
+      "squirrel monkey, Saimiri sciureus": 382,
+      "stage": 819,
+      "standard poodle": 267,
+      "standard schnauzer": 198,
+      "starfish, sea star": 327,
+      "steam locomotive": 820,
+      "steel arch bridge": 821,
+      "steel drum": 822,
+      "stethoscope": 823,
+      "stingray": 6,
+      "stinkhorn, carrion fungus": 994,
+      "stole": 824,
+      "stone wall": 825,
+      "stopwatch, stop watch": 826,
+      "stove": 827,
+      "strainer": 828,
+      "strawberry": 949,
+      "street sign": 919,
+      "streetcar, tram, tramcar, trolley, trolley car": 829,
+      "stretcher": 830,
+      "studio couch, day bed": 831,
+      "stupa, tope": 832,
+      "sturgeon": 394,
+      "submarine, pigboat, sub, U-boat": 833,
+      "suit, suit of clothes": 834,
+      "sulphur butterfly, sulfur butterfly": 325,
+      "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita": 89,
+      "sundial": 835,
+      "sunglass": 836,
+      "sunglasses, dark glasses, shades": 837,
+      "sunscreen, sunblock, sun blocker": 838,
+      "suspension bridge": 839,
+      "swab, swob, mop": 840,
+      "sweatshirt": 841,
+      "swimming trunks, bathing trunks": 842,
+      "swing": 843,
+      "switch, electric switch, electrical switch": 844,
+      "syringe": 845,
+      "tabby, tabby cat": 281,
+      "table lamp": 846,
+      "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui": 32,
+      "tank, army tank, armored combat vehicle, armoured combat vehicle": 847,
+      "tape player": 848,
+      "tarantula": 76,
+      "teapot": 849,
+      "teddy, teddy bear": 850,
+      "television, television system": 851,
+      "tench, Tinca tinca": 0,
+      "tennis ball": 852,
+      "terrapin": 36,
+      "thatch, thatched roof": 853,
+      "theater curtain, theatre curtain": 854,
+      "thimble": 855,
+      "three-toed sloth, ai, Bradypus tridactylus": 364,
+      "thresher, thrasher, threshing machine": 856,
+      "throne": 857,
+      "thunder snake, worm snake, Carphophis amoenus": 52,
+      "tick": 78,
+      "tiger beetle": 300,
+      "tiger cat": 282,
+      "tiger shark, Galeocerdo cuvieri": 3,
+      "tiger, Panthera tigris": 292,
+      "tile roof": 858,
+      "timber wolf, grey wolf, gray wolf, Canis lupus": 269,
+      "titi, titi monkey": 380,
+      "toaster": 859,
+      "tobacco shop, tobacconist shop, tobacconist": 860,
+      "toilet seat": 861,
+      "toilet tissue, toilet paper, bathroom tissue": 999,
+      "torch": 862,
+      "totem pole": 863,
+      "toucan": 96,
+      "tow truck, tow car, wrecker": 864,
+      "toy poodle": 265,
+      "toy terrier": 158,
+      "toyshop": 865,
+      "tractor": 866,
+      "traffic light, traffic signal, stoplight": 920,
+      "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi": 867,
+      "tray": 868,
+      "tree frog, tree-frog": 31,
+      "trench coat": 869,
+      "triceratops": 51,
+      "tricycle, trike, velocipede": 870,
+      "trifle": 927,
+      "trilobite": 69,
+      "trimaran": 871,
+      "tripod": 872,
+      "triumphal arch": 873,
+      "trolleybus, trolley coach, trackless trolley": 874,
+      "trombone": 875,
+      "tub, vat": 876,
+      "turnstile": 877,
+      "tusker": 101,
+      "typewriter keyboard": 878,
+      "umbrella": 879,
+      "unicycle, monocycle": 880,
+      "upright, upright piano": 881,
+      "vacuum, vacuum cleaner": 882,
+      "valley, vale": 979,
+      "vase": 883,
+      "vault": 884,
+      "velvet": 885,
+      "vending machine": 886,
+      "vestment": 887,
+      "viaduct": 888,
+      "vine snake": 59,
+      "violin, fiddle": 889,
+      "vizsla, Hungarian pointer": 211,
+      "volcano": 980,
+      "volleyball": 890,
+      "vulture": 23,
+      "waffle iron": 891,
+      "walking stick, walkingstick, stick insect": 313,
+      "wall clock": 892,
+      "wallaby, brush kangaroo": 104,
+      "wallet, billfold, notecase, pocketbook": 893,
+      "wardrobe, closet, press": 894,
+      "warplane, military plane": 895,
+      "warthog": 343,
+      "washbasin, handbasin, washbowl, lavabo, wash-hand basin": 896,
+      "washer, automatic washer, washing machine": 897,
+      "water bottle": 898,
+      "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis": 346,
+      "water jug": 899,
+      "water ouzel, dipper": 20,
+      "water snake": 58,
+      "water tower": 900,
+      "weasel": 356,
+      "web site, website, internet site, site": 916,
+      "weevil": 307,
+      "whippet": 172,
+      "whiptail, whiptail lizard": 41,
+      "whiskey jug": 901,
+      "whistle": 902,
+      "white stork, Ciconia ciconia": 127,
+      "white wolf, Arctic wolf, Canis lupus tundrarum": 270,
+      "wig": 903,
+      "wild boar, boar, Sus scrofa": 342,
+      "window screen": 904,
+      "window shade": 905,
+      "wine bottle": 907,
+      "wing": 908,
+      "wire-haired fox terrier": 188,
+      "wok": 909,
+      "wolf spider, hunting spider": 77,
+      "wombat": 106,
+      "wood rabbit, cottontail, cottontail rabbit": 330,
+      "wooden spoon": 910,
+      "wool, woolen, woollen": 911,
+      "worm fence, snake fence, snake-rail fence, Virginia fence": 912,
+      "wreck": 913,
+      "yawl": 914,
+      "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum": 986,
+      "yurt": 915,
+      "zebra": 340,
+      "zucchini, courgette": 939
+    },
+    "model_type": "swinv2",
+    "num_heads": [
+      6,
+      12,
+      24,
+      48
+    ],
+    "path_norm": true,
+    "torch_dtype": "float32",
+    "window_size": 16
+  },
+  "eos_token_id": 6,
+  "is_encoder_decoder": true,
+  "model_type": "clip-encoder-decoder",
+  "pad_token_id": 6,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2"
+}

configuration_clipcap.py

@@ -0,0 +1,30 @@
+from transformers import PretrainedConfig, AutoConfig
+
+
+class CLIPEncoderDecoderConfig(PretrainedConfig):
+    model_type = "clip-encoder-decoder"
+
+    def __init__(
+        self, 
+        **kwargs):
+        super().__init__(**kwargs)
+
+        self.encoder = AutoConfig.from_pretrained('microsoft/swinv2-large-patch4-window12to16-192to256-22kto1k-ft')
+        self.decoder = AutoConfig.from_pretrained('airesearch/wangchanberta-base-att-spm-uncased')
+        self.is_encoder_decoder = True
+
+    @classmethod
+    def from_encoder_decoder_configs(
+            cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
+    ) -> PretrainedConfig:
+        r"""
+        Instantiate a [`VisionEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
+        configuration and decoder model configuration.
+
+        Returns:
+            [`VisionEncoderDecoderConfig`]: An instance of a configuration object
+        """
+        decoder_config.is_decoder = True
+        decoder_config.add_cross_attention = True
+
+        return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.36.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6fb23f1bbc92007e9a6d3807c1522575cabf646665b8133362e85d83be9f99a
+size 964407452

modeling_clipcap.py

@@ -0,0 +1,292 @@
+from transformers import (
+    PreTrainedModel,
+    VisionEncoderDecoderModel,
+    VisionEncoderDecoderConfig,
+    AutoModel,
+    AutoModelForCausalLM,
+    AutoConfig
+)
+from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput
+from torch import nn
+from .configuration_clipcap import CLIPEncoderDecoderConfig
+from typing import Optional, Tuple, Union
+import torch
+import gc
+import os
+import tempfile
+
+
+def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    if decoder_start_token_id is None:
+        raise ValueError("Make sure to set the decoder_start_token_id attribute of the model's configuration.")
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("Make sure to set the pad_token_id attribute of the model's configuration.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+
+class Encoder(nn.Module):
+    main_input_name = 'pixel_values'
+    def __init__(self):
+        super().__init__()
+        clip = AutoModel.from_pretrained('openai/clip-vit-base-patch32')
+        self.vision_model = clip.vision_model
+        self.visual_projection = clip.visual_projection
+        self.config = clip.vision_model.config
+        self.config.hidden_size = clip.config.projection_dim
+
+    def forward(self, pixel_values, output_attentions=None, output_hidden_states=None, return_dict=False, **kwargs):
+        vision_outputs = self.vision_model(
+            pixel_values=pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        pooled_output = vision_outputs[1]  # pooled_output
+        image_features = self.visual_projection(pooled_output).view(pooled_output.size(0), 1, -1)
+        return BaseModelOutput(last_hidden_state=image_features)
+    def get_output_embeddings(self):
+        pass
+
+class CLIPEncoderDecoderModel(PreTrainedModel):
+    config_class = CLIPEncoderDecoderConfig
+    base_model_prefix = "clip_encoder_decoder"
+    main_input_name = "pixel_values"
+    supports_gradient_checkpointing = True
+    def __init__(
+        self,
+        config = None,
+        encoder = None,
+        decoder = None,
+    ):
+        config.tie_word_embeddings = False
+        super().__init__(config)
+
+        encoder = Encoder()
+        encoder_hidden_size = encoder.config.hidden_size
+
+        if decoder is None:
+            config.decoder.is_decoder = True
+            config.decoder.add_cross_attention = True
+            decoder = AutoModelForCausalLM.from_config(config.decoder)
+
+        self.encoder = encoder
+        self.decoder = decoder
+
+        self.encoder.config = self.config.encoder
+        self.decoder.config = self.config.decoder
+
+        self.enc_to_dec_proj = nn.Linear(encoder_hidden_size, self.decoder.config.hidden_size)
+
+    def get_encoder(self):
+        return self.encoder
+
+    def get_decoder(self):
+        return self.decoder
+
+    def get_output_embeddings(self):
+        return self.decoder.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        return self.decoder.set_output_embeddings(new_embeddings)
+
+    @classmethod
+    def from_encoder_decoder_pretrained(
+        cls,
+        encoder_pretrained_model_name_or_path: str = None,
+        decoder_pretrained_model_name_or_path: str = None,
+        *model_args,
+        **kwargs,
+    ) -> PreTrainedModel:
+        kwargs_encoder = {
+            argument[len("encoder_") :]: value for argument, value in kwargs.items() if argument.startswith("encoder_")
+        }
+
+        kwargs_decoder = {
+            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
+        }
+
+        # remove encoder, decoder kwargs from kwargs
+        for key in kwargs_encoder.keys():
+            del kwargs["encoder_" + key]
+        for key in kwargs_decoder.keys():
+            del kwargs["decoder_" + key]
+
+        # Load and initialize the encoder and decoder
+        # The distinction between encoder and decoder at the model level is made
+        # by the value of the flag `is_decoder` that we need to set correctly.
+        encoder = kwargs_encoder.pop("model", None)
+        if encoder is None:
+            if encoder_pretrained_model_name_or_path is None:
+                raise ValueError(
+                    "If `encoder_model` is not defined as an argument, a `encoder_pretrained_model_name_or_path` has "
+                    "to be defined."
+                )
+
+            if "config" not in kwargs_encoder:
+                encoder_config, kwargs_encoder = AutoConfig.from_pretrained(
+                    encoder_pretrained_model_name_or_path, **kwargs_encoder, return_unused_kwargs=True
+                )
+
+                if encoder_config.is_decoder is True or encoder_config.add_cross_attention is True:
+                    encoder_config.is_decoder = False
+                    encoder_config.add_cross_attention = False
+
+                kwargs_encoder["config"] = encoder_config
+
+            encoder = AutoModel.from_pretrained(encoder_pretrained_model_name_or_path, *model_args, **kwargs_encoder)
+
+        decoder = kwargs_decoder.pop("model", None)
+        if decoder is None:
+            if decoder_pretrained_model_name_or_path is None:
+                raise ValueError(
+                    "If `decoder_model` is not defined as an argument, a `decoder_pretrained_model_name_or_path` has "
+                    "to be defined."
+                )
+
+            if "config" not in kwargs_decoder:
+                decoder_config, kwargs_decoder = AutoConfig.from_pretrained(
+                    decoder_pretrained_model_name_or_path, **kwargs_decoder, return_unused_kwargs=True
+                )
+
+                if decoder_config.is_decoder is False or decoder_config.add_cross_attention is False:
+                    decoder_config.is_decoder = True
+                    decoder_config.add_cross_attention = True
+
+                kwargs_decoder["config"] = decoder_config
+
+            decoder = AutoModelForCausalLM.from_pretrained(decoder_pretrained_model_name_or_path, **kwargs_decoder)
+
+        # instantiate config with corresponding kwargs
+        config = VisionEncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config, **kwargs)
+
+        # make sure input & output embeddings is not tied
+        config.tie_word_embeddings = False
+        return cls(encoder=encoder, decoder=decoder, config=config)
+
+    def forward(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        decoder_input_ids: Optional[torch.LongTensor] = None,
+        decoder_attention_mask: Optional[torch.BoolTensor] = None,
+        encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        decoder_inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        kwargs_encoder = {argument: value for argument, value in kwargs.items() if not argument.startswith("decoder_")}
+
+        kwargs_decoder = {
+            argument[len("decoder_") :]: value for argument, value in kwargs.items() if argument.startswith("decoder_")
+        }
+
+        if encoder_outputs is None:
+            if pixel_values is None:
+                raise ValueError("You have to specify pixel_values")
+
+            encoder_outputs = self.encoder(
+                pixel_values,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                **kwargs_encoder,
+            )
+        elif isinstance(encoder_outputs, tuple):
+            encoder_outputs = BaseModelOutput(*encoder_outputs)
+
+        encoder_hidden_states = encoder_outputs[0]
+
+        encoder_hidden_states = self.enc_to_dec_proj(encoder_hidden_states)
+
+        # else:
+        encoder_attention_mask = None
+
+        if (labels is not None) and (decoder_input_ids is None and decoder_inputs_embeds is None):
+            decoder_input_ids = shift_tokens_right(
+                labels, self.config.pad_token_id, self.config.decoder_start_token_id
+            )
+
+        # Decode
+        decoder_outputs = self.decoder(
+            input_ids=decoder_input_ids,
+            attention_mask=decoder_attention_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            inputs_embeds=decoder_inputs_embeds,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            use_cache=use_cache,
+            past_key_values=past_key_values,
+            return_dict=return_dict,
+            **kwargs_decoder,
+        )
+
+        # Compute loss independent from decoder (as some shift the logits inside them)
+        loss = None
+        if labels is not None:
+            logits = decoder_outputs.logits if return_dict else decoder_outputs[0]
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits.reshape(-1, self.decoder.config.vocab_size), labels.reshape(-1))
+
+        if not return_dict:
+            if loss is not None:
+                return (loss,) + decoder_outputs + encoder_outputs
+            else:
+                return decoder_outputs + encoder_outputs
+
+        return Seq2SeqLMOutput(
+            loss=loss,
+            logits=decoder_outputs.logits,
+            past_key_values=decoder_outputs.past_key_values,
+            decoder_hidden_states=decoder_outputs.hidden_states,
+            decoder_attentions=decoder_outputs.attentions,
+            cross_attentions=decoder_outputs.cross_attentions,
+            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
+            encoder_hidden_states=encoder_outputs.hidden_states,
+            encoder_attentions=encoder_outputs.attentions,
+        )
+
+    def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
+        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, use_cache=None, encoder_outputs=None, **kwargs
+    ):
+        decoder_inputs = self.decoder.prepare_inputs_for_generation(input_ids, past_key_values=past_key_values)
+        decoder_attention_mask = decoder_inputs["attention_mask"] if "attention_mask" in decoder_inputs else None
+        input_dict = {
+            "attention_mask": attention_mask,
+            "decoder_attention_mask": decoder_attention_mask,
+            "decoder_input_ids": decoder_inputs["input_ids"],
+            "encoder_outputs": encoder_outputs,
+            "past_key_values": decoder_inputs["past_key_values"],
+            "use_cache": use_cache,
+        }
+        return input_dict
+
+    def resize_token_embeddings(self, *args, **kwargs):
+        raise NotImplementedError(
+            "Resizing the embedding layers via the VisionEncoderDecoderModel directly is not supported.Please use the"
+            " respective methods of the wrapped decoder object (model.decoder.resize_token_embeddings(...))"
+        )
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        # apply decoder cache reordering here
+        return self.decoder._reorder_cache(past_key_values, beam_idx)

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c4ba4e495ddf31eb2fdba7fc6aef3c233091d25d35bc9d24694ccf48ae114c
+size 904693

special_tokens_map.json

@@ -0,0 +1,56 @@
+{
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED",
+    "<_>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8139dded1a09a437bf2a7608d71977eec5e97141fb054729cabae88c20bb5c
+size 2177870

tokenizer_config.json

@@ -0,0 +1,84 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>NOTUSED",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>NOTUSED",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<_>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "25004": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<s>NOTUSED",
+    "</s>NOTUSED",
+    "<_>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "CamembertTokenizer",
+  "unk_token": "<unk>"
+}