Upload 56 files

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DeepSeek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

coding_problems.py

@@ -0,0 +1,201 @@
+coding_problems = {
+    "Python": "In Python, write a function to reverse a string, for example, turning input 'hello' into 'olleh'.",
+    "Java": "In Java, implement a simple calculator class that supports addition, subtraction, multiplication, and division.",
+    "C++": "In C++, implement an efficient sorting algorithm (e.g., quicksort or mergesort) to sort an array of integers.",
+    "JavaScript": "In JavaScript, create a simple web page interaction with a button that, when clicked, displays the current time.",
+    "C#": "In C#, implement a simple Windows Forms application featuring a text box and a button that displays the text box content when clicked.",
+    "Ruby": "In Ruby, implement a simple web server using Ruby on Rails that returns 'Hello, World!'",
+    "PHP": "In PHP, create a dynamic web page that displays the current date and time.",
+    "Swift": "In Swift, implement a simple iOS app interface with a label and a button that updates the label to 'Clicked!' when pressed.",
+    "Go": "In Go, implement a concurrent program using goroutines to calculate the square and cube of two numbers simultaneously.",
+    "R": "In R, read data from a CSV file, calculate the average, and plot a bar chart.",
+    "SQL": "In SQL, write a query to retrieve the total number of orders for each user from 'users' and 'orders' tables.",
+    "Kotlin": "In Kotlin, implement a simple Android app feature with an input field and a button that displays the input content when clicked.",
+    "Scala": "In Scala, implement a functional programming example with a function that takes a list of integers and returns their sum.",
+    "Perl": "In Perl, implement a text processing script to extract all email addresses from a text file.",
+    "Haskell": "In Haskell, implement a pure functional program to calculate the nth Fibonacci number.",
+    "Lua": "In Lua, implement a game script to control a character's simple movements (e.g., up, down, left, right).",
+    "MATLAB": "In MATLAB, implement matrix multiplication and visualize the result.",
+    "Objective-C": "In Objective-C, implement a simple iOS app feature with a button that shows an alert with 'Welcome!' when clicked.",
+    "Rust": "In Rust, implement a safe concurrent program to calculate the sum of all elements in an array using multiple threads.",
+    "TypeScript": "In TypeScript, implement a type-safe function that takes an array of numbers and returns the maximum value.",
+    "Bash": "In Bash, write a shell script to backup a specified directory to a compressed file.",
+    "PowerShell": "In PowerShell, implement a system management script to list all running processes.",
+    "Dart": "In Dart, implement a simple Flutter app interface with a counter button.",
+    "Groovy": "In Groovy, implement a simple script to read and print the content of a JSON file.",
+    "F#": "In F#, implement a functional programming example with a function to calculate the product of all elements in an integer list.",
+    "Erlang": "In Erlang, implement a simple distributed chat server that supports multiple client connections.",
+    "COBOL": "In COBOL, implement a business calculation program to compute an employee's total salary (base salary + bonus).",
+    "Fortran": "In Fortran, implement a scientific computing program to calculate a definite integral (e.g., using the rectangle method).",
+    "Ada": "In Ada, implement an embedded system program to simulate controlling an LED light (on/off).",
+    "Prolog": "In Prolog, implement a logic reasoning program to solve a simple puzzle, such as 'Who is the murderer?'",
+    "Lisp": "In Lisp, implement a symbolic computation program to evaluate a simple mathematical expression, e.g., '(add 2 3)'.",
+    "Scheme": "In Scheme, implement a recursive function to calculate the factorial of n.",
+    "Smalltalk": "In Smalltalk, implement an object-oriented program to simulate bank account deposits and withdrawals.",
+    "VHDL": "In VHDL, design a simple digital circuit, such as an adder.",
+    "Verilog": "In Verilog, implement a hardware module for a simple sequential logic circuit, like a counter.",
+    "Assembly": "In Assembly, write a low-level program to compute the sum of two numbers.",
+    "Clojure": "In Clojure, implement a concurrent program using concurrency primitives to compute the sum of squares of an array.",
+    "Elixir": "In Elixir, implement a simple distributed counter.",
+    "Julia": "In Julia, implement a high-performance computing program for matrix multiplication and measure execution time.",
+    "Crystal": "In Crystal, implement a web application that returns a JSON response.",
+    "Nim": "In Nim, implement a command-line tool that accepts arguments and prints the content of a file.",
+    "D": "In D, implement a system-level program to list all files in a directory.",
+    "Pascal": "In Pascal, implement a teaching program demonstrating if-else and loop structures.",
+    "Forth": "In Forth, implement a stack-based addition calculator.",
+    "APL": "In APL, implement an array processing program to sort an array and calculate its average.",
+    "J": "In J, implement a data analysis program to compute the standard deviation of a dataset.",
+    "K": "In K, implement a high-performance data processing program to compute the sum of a large array.",
+    "Q": "In Q, implement a database query to retrieve data from a time-series database for a specific period.",
+    "Tcl": "In Tcl, implement a script to automate file renaming.",
+    "Awk": "In Awk, implement a text processing script to extract the second field from each line of a log file.",
+    "Sed": "In Sed, implement a text replacement script to replace 'cat' with 'dog' in a text file.",
+    "Grep": "In Grep, implement a pattern matching script to find lines containing 'error' in a file.",
+    "Makefile": "In Makefile, write a build script to compile a C program and generate an executable file.",
+    "CMake": "In CMake, implement a cross-platform build system configuration.",
+    "Dockerfile": "In Dockerfile, write a container configuration file to build a Docker image with a Python environment.",
+    "YAML": "In YAML, implement a configuration file for a web application, specifying port and database connection.",
+    "JSON": "In JSON, implement a data exchange format to represent a user's personal information.",
+    "XML": "In XML, implement a markup language document to describe a book's title and author.",
+    "HTML": "In HTML, implement a simple web page with a title and a paragraph.",
+    "CSS": "In CSS, implement a style sheet to add background color and font styles to an HTML page.",
+    "Sass": "In Sass, implement a CSS preprocessor script to generate nested style sheets.",
+    "Less": "In Less, implement a CSS preprocessor script using variables to define colors and generate CSS.",
+    "PostScript": "In PostScript, implement a print script to generate a circle graphic.",
+    "TeX": "In TeX, implement a typesetting document that includes mathematical formulas.",
+    "ABAP": "In ABAP, implement a report program to retrieve and display customer data from an SAP database.",
+    "ActionScript": "In ActionScript, create an interactive animation where a ball moves across the screen when a button is clicked.",
+    "ALGOL": "In ALGOL, implement a procedure to calculate the greatest common divisor (GCD) of two numbers.",
+    "Ballerina": "In Ballerina, implement a simple REST API service that returns a greeting message.",
+    "Basic": "In Basic, write a program to calculate and display the area of a rectangle based on user input.",
+    "BCPL": "In BCPL, implement a simple memory allocation function for dynamic arrays.",
+    "Chapel": "In Chapel, implement a parallel program to compute the dot product of two large vectors.",
+    "ColdFusion": "In ColdFusion, create a dynamic webpage that displays a list of products from a database.",
+    "Datalog": "In Datalog, write a query to determine family relationships (e.g., 'Who is the grandparent?') from a set of facts.",
+    "Elm": "In Elm, implement a simple counter web app with increment and decrement buttons.",
+    "Hack": "In Hack, implement a type-safe function to validate and process a user registration form.",
+    "Icon": "In Icon, write a program to generate all permutations of a given string.",
+    "Io": "In Io, implement a prototype-based object system to simulate a simple car with start and stop methods.",
+    "J#": "In J#, implement a console application to sort a list of names alphabetically.",
+    "LabVIEW": "In LabVIEW, create a virtual instrument to measure and display the voltage from a simulated sensor.",
+    "Ladder Logic": "In Ladder Logic, design a PLC program to control a traffic light sequence.",
+    "LiveScript": "In LiveScript, implement a functional pipeline to process and filter an array of numbers.",
+    "Logo": "In Logo, write a script to draw a square using turtle graphics.",
+    "Mercury": "In Mercury, implement a type-safe program to solve a Sudoku puzzle.",
+    "Modula-2": "In Modula-2, implement a module to manage a simple inventory system.",
+    "OCaml": "In OCaml, implement a function to balance a binary search tree.",
+    "PL/I": "In PL/I, write a program to calculate monthly loan payments based on principal and interest rate.",
+    "Processing": "In Processing, create a visual sketch where circles follow the mouse cursor.",
+    "Puppet": "In Puppet, write a manifest to configure a web server with specific ports and packages.",
+    "Racket": "In Racket, implement a simple interpreter for a custom arithmetic language.",
+    "REXX": "In REXX, write a script to parse and reformat a log file into a report.",
+    "SAS": "In SAS, implement a data analysis program to compute summary statistics from a dataset.",
+    "Scratch": "In Scratch, create a simple game where a sprite moves to avoid obstacles.",
+    "Solidity": "In Solidity, implement a smart contract for a voting system on the Ethereum blockchain.",
+    "SPARK": "In SPARK, implement a verified program to control an elevator��s floor selection safely.",
+    "Squirrel": "In Squirrel, write a script to manage a simple game character��s health and inventory.",
+    "Stan": "In Stan, implement a Bayesian model to fit a linear regression to sample data.",
+    "Vala": "In Vala, implement a GTK application with a window and a button that changes color when clicked.",
+    "VBA": "In VBA, automate an Excel task to sum a column and highlight cells above a threshold.",
+    "Zig": "In Zig, implement a memory-safe program to reverse an array without using extra space.",
+    "Apex": "In Apex, implement a Salesforce trigger to automatically update a contact��s status when an opportunity is closed.",
+    "B": "In B, write a low-level function to swap two integer values without using a temporary variable.",
+    "Brainfuck": "In Brainfuck, implement a program to print 'Hello, World!' using its minimalistic syntax.",
+    "Ceylon": "In Ceylon, implement a type-safe class hierarchy for a library system with books and borrowers.",
+    "CFML": "In CFML, create a dynamic page to display a user��s profile fetched from a database.",
+    "Chisel": "In Chisel, design a hardware module for a simple 4-bit counter.",
+    "Clarion": "In Clarion, implement a database application to manage customer orders and invoices.",
+    "Clean": "In Clean, implement a purely functional program to compute the greatest common divisor (GCD) of two numbers.",
+    "Clipper": "In Clipper, write a program to manage a simple inventory database with add and delete functions.",
+    "Common Lisp": "In Common Lisp, implement a symbolic differentiation function for basic algebraic expressions.",
+    "Dafny": "In Dafny, implement a verified program to sort an array with provable correctness.",
+    "Delphi": "In Delphi, create a desktop application with a form to calculate and display a factorial.",
+    "Eiffel": "In Eiffel, implement a class with design-by-contract to manage a stack with push and pop operations.",
+    "Elm": "In Elm, build a simple web app with a text input that displays the character count in real-time.",
+    "Factor": "In Factor, implement a stack-based program to calculate the sum of numbers from 1 to n.",
+    "Falcon": "In Falcon, write a script to process and format a CSV file into a readable report.",
+    "Frink": "In Frink, implement a unit conversion program to convert between miles and kilometers.",
+    "GAMS": "In GAMS, model an optimization problem to minimize the cost of a production process.",
+    "GAP": "In GAP, implement a group theory computation to find the order of a permutation group.",
+    "Genie": "In Genie, create a simple GTK application with a button that toggles a label��s text.",
+    "Haxe": "In Haxe, implement a cross-platform program to draw a animated bouncing ball.",
+    "Hy": "In Hy, write a Lisp-style program to filter even numbers from a list.",
+    "Idris": "In Idris, implement a dependently-typed function to append two lists with length preservation.",
+    "Inform": "In Inform, create a simple text adventure game where the player navigates a room.",
+    "JCL": "In JCL, write a job control script to execute a COBOL program on a mainframe.",
+    "Jython": "In Jython, implement a Python script to interact with a Java library and print system properties.",
+    "KRL": "In KRL, design a rule-based system to send a notification when a sensor value exceeds a threshold.",
+    "LFE": "In LFE, implement a Lisp-based concurrent program to calculate factorials using actors.",
+    "MQL4": "In MQL4, write a trading script for MetaTrader to calculate moving averages of stock prices.",
+    "Nemerle": "In Nemerle, implement a macro-based program to generate boilerplate code for a class.",
+    "Oberon": "In Oberon, write a modular program to simulate a simple file system.",
+    "Opa": "In Opa, implement a full-stack web app with a chat interface running on both client and server.",
+    "OpenCL": "In OpenCL, write a parallel program to perform matrix addition on a GPU.",
+    "Oz": "In Oz, implement a multi-paradigm program to solve a constraint satisfaction problem like Sudoku.",
+    "Pike": "In Pike, create a simple network server that echoes back client messages.",
+    "Pony": "In Pony, implement an actor-based program to simulate a bank with concurrent transactions.",
+    "Pure": "In Pure, write a term-rewriting program to simplify algebraic expressions.",
+    "Q#": "In Q#, implement a quantum algorithm to generate a random bit using superposition.",
+    "Red": "In Red, create a lightweight GUI app with a button that changes the window��s title.",
+    "Ring": "In Ring, implement a simple game loop to move a character across the screen.",
+    "RPG": "In RPG, write a business program to calculate employee payroll with tax deductions.",
+    "S-Lang": "In S-Lang, implement a script to process astronomical data and compute averages.",
+    "Simula": "In Simula, simulate a queueing system for customers in a bank.",
+    "Standard ML": "In Standard ML, implement a functional program to reverse a list using pattern matching.",
+    "SuperCollider": "In SuperCollider, create a synthesizer script to generate a sine wave sound.",
+    "Turing": "In Turing, write an educational program to draw a spiral using turtle graphics.",
+    "UnrealScript": "In UnrealScript, implement a simple game mechanic to spawn an enemy in Unreal Engine.",
+    "Wolfram Language": "In Wolfram Language, create a visualization of a 3D mathematical function.",
+    "X10": "In X10, implement a distributed program to compute the sum of an array across multiple nodes.",
+    "ZPL": "In ZPL, write a parallel program to perform a stencil computation on a 2D grid.",
+    "A#": "In A#, implement a simple audio processing script to generate a square wave tone.",
+    "ABC": "In ABC, write a program to calculate and display the first 10 numbers of the Fibonacci sequence.",
+    "Accent": "In Accent, implement a reactive system to update a counter based on incoming events.",
+    "Agora": "In Agora, create an object-oriented program to simulate a simple online auction.",
+    "Alef": "In Alef, implement a concurrent program to manage multiple threads printing numbers.",
+    "Alice": "In Alice, design a 3D animation where a character walks across a scene.",
+    "Amiga E": "In Amiga E, write a program to draw a colorful pattern on an Amiga-style display.",
+    "AngelScript": "In AngelScript, implement a game scripting function to handle player movement in a 2D game.",
+    "APL2": "In APL2, write an array-based program to compute the transpose of a matrix.",
+    "Arc": "In Arc, implement a minimal web server that serves a custom HTML page.",
+    "AspectJ": "In AspectJ, add logging functionality to an existing Java class using aspects.",
+    "Ateji PX": "In Ateji PX, implement a parallel program to compute the sum of a large array.",
+    "AutoHotkey": "In AutoHotkey, create a script to automate repetitive keyboard and mouse actions.",
+    "Befunge": "In Befunge, write a 2D program to print the numbers from 1 to 10.",
+    "BLISS": "In BLISS, implement a low-level routine to optimize bit manipulation in a system program.",
+    "Boo": "In Boo, create a simple scripting program to process and sort a list of strings.",
+    "C--": "In C--, implement an intermediate language function to multiply two integers.",
+    "Caml": "In Caml, write a functional program to compute the depth of a binary tree.",
+    "Charity": "In Charity, implement a purely functional program to process a stream of integers.",
+    "ChucK": "In ChucK, create a real-time audio program to play a repeating drum pattern.",
+    "Cilk": "In Cilk, implement a parallel program to compute the factorial of a number using spawn and sync.",
+    "COMAL": "In COMAL, write an educational program to teach loops by printing a multiplication table.",
+    "Curl": "In Curl, implement a rich internet application with a clickable button that changes text.",
+    "DASL": "In DASL, design a dataflow program to filter and transform a stream of sensor data.",
+    "DIBOL": "In DIBOL, write a business application to calculate total sales from a list of transactions.",
+    "E": "In E, implement a secure distributed system to exchange encrypted messages between two nodes.",
+    "Ease": "In Ease, create a visual programming script to animate a bouncing ball.",
+    "Euphoria": "In Euphoria, write a simple interpreter for a custom command language.",
+    "Fantom": "In Fantom, implement a cross-platform program to read and display a configuration file.",
+    "Ferite": "In Ferite, create a lightweight script to simulate a dice-rolling game.",
+    "FL": "In FL, implement a functional logic program to solve a simple puzzle like Towers of Hanoi.",
+    "Focal": "In Focal, write a scientific calculator program to compute trigonometric functions.",
+    "GameMaker Language": "In GameMaker Language, implement a simple platformer mechanic for jumping.",
+    "GML": "In GML, design a graph manipulation program to find the shortest path between nodes.",
+    "Hope": "In Hope, implement a lazy functional program to generate an infinite list of primes.",
+    "ICI": "In ICI, write a script to process and reformat a text file into a table.",
+    "INTERCAL": "In INTERCAL, create a humorous program to compute '2 + 2' with unnecessary complexity.",
+    "Joy": "In Joy, implement a stack-based program to reverse a sequence of numbers.",
+    "Kodu": "In Kodu, design a simple game where a character collects objects in a 3D world.",
+    "Leda": "In Leda, write a multi-paradigm program combining imperative and functional styles to sort a list.",
+    "Limbo": "In Limbo, implement a concurrent program for a distributed system running on Inferno OS.",
+    "Miranda": "In Miranda, create a lazy functional program to compute the sum of squares up to n.",
+    "MUMPS": "In MUMPS, implement a database application to store and retrieve patient records.",
+    "Nial": "In Nial, write an array-oriented program to compute the average of nested arrays.",
+    "Obliq": "In Obliq, implement a distributed GUI application with a button that updates remotely.",
+    "OmniMark": "In OmniMark, create a text-processing script to convert XML to HTML.",
+    "Opal": "In Opal, implement a functional program to manipulate algebraic data types like lists.",
+    "PILOT": "In PILOT, write an educational script to quiz users on basic math problems.",
+    "Self": "In Self, implement a prototype-based object system to model a simple car with behaviors.",
+    "TOM": "In TOM, create a pattern-matching program to transform a string based on rules."    	    	
+}

config.json

@@ -0,0 +1,61 @@
+{
+  "architectures": [
+    "DeepseekV3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+    "AutoModel": "modeling_deepseek.DeepseekV3Model",
+    "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+  },
+  "aux_loss_alpha": 0.001,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "ep_size": 1,
+  "first_k_dense_replace": 3,
+  "hidden_act": "silu",
+  "hidden_size": 7168,
+  "initializer_range": 0.02,
+  "intermediate_size": 18432,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 163840,
+  "model_type": "deepseek_v3",
+  "moe_intermediate_size": 2048,
+  "moe_layer_freq": 1,
+  "n_group": 8,
+  "n_routed_experts": 160,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 128,
+  "num_experts_per_tok": 6,
+  "num_hidden_layers": 61,
+  "num_key_value_heads": 128,
+  "num_nextn_predict_layers": 1,
+  "pretraining_tp": 1,
+  "q_lora_rank": 1536,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "beta_fast": 32,
+    "beta_slow": 1,
+    "factor": 40,
+    "mscale": 1.0,
+    "mscale_all_dim": 1.0,
+    "original_max_position_embeddings": 4096,
+    "type": "yarn"
+  },
+  "rope_theta": 10000,
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "seq_aux": true,
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "topk_method": "noaux_tc",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "v_head_dim": 128,
+  "vocab_size": 129280
+}

configuration_deepseek.py

@@ -0,0 +1,210 @@
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+class DeepseekV3Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`DeepseekV3Model`]. It is used to instantiate an DeepSeek
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the DeepSeek-V3.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 129280):
+            Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`DeepseekV3Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        moe_intermediate_size (`int`, *optional*, defaults to 1407):
+            Dimension of the MoE representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_nextn_predict_layers (`int`, *optional*, defaults to 1):
+            Number of nextn predict layers in the DeepSeekV3 Model.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        n_shared_experts (`int`, *optional*, defaults to None):
+            Number of shared experts, None means dense model.
+        n_routed_experts (`int`, *optional*, defaults to None):
+            Number of routed experts, None means dense model.
+        routed_scaling_factor (`float`, *optional*, defaults to 1.0):
+            Scaling factor or routed experts.
+        topk_method (`str`, *optional*, defaults to `gready`):
+            Topk method used in routed gate.
+        n_group (`int`, *optional*, defaults to None):
+            Number of groups for routed experts.
+        topk_group (`int`, *optional*, defaults to None):
+            Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
+        num_experts_per_tok (`int`, *optional*, defaults to None):
+            Number of selected experts, None means dense model.
+        moe_layer_freq (`int`, *optional*, defaults to 1):
+            The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
+        first_k_dense_replace (`int`, *optional*, defaults to 0):
+            Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
+                                                            \--k dense layers--/
+        norm_topk_prob (`bool`, *optional*, defaults to False):
+            Whether to normalize the weights of the routed experts.
+        scoring_func (`str`, *optional*, defaults to 'softmax'):
+            Method of computing expert weights.
+        aux_loss_alpha (`float`, *optional*, defaults to 0.001):
+            Auxiliary loss weight coefficient.
+        seq_aux = (`bool`, *optional*, defaults to True):
+            Whether to compute the auxiliary loss for each individual sample.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
+            necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
+            issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum.
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+
+    ```python
+    >>> from transformers import DeepseekV3Model, DeepseekV3Config
+
+    >>> # Initializing a Deepseek-V3 style configuration
+    >>> configuration = DeepseekV3Config()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "deepseek_v3"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=129280,
+        hidden_size=7168,
+        intermediate_size=18432,
+        moe_intermediate_size = 2048,
+        num_hidden_layers=61,
+        num_nextn_predict_layers=1,
+        num_attention_heads=128,
+        num_key_value_heads=128,
+        n_shared_experts = 1,
+        n_routed_experts = 256,
+        ep_size = 1,
+        routed_scaling_factor = 2.5,
+        kv_lora_rank = 512,
+        q_lora_rank = 1536,
+        qk_rope_head_dim = 64,
+        v_head_dim = 128,
+        qk_nope_head_dim = 128,
+        topk_method = 'noaux_tc',
+        n_group = 8,
+        topk_group = 4,
+        num_experts_per_tok = 8,
+        moe_layer_freq = 1,
+        first_k_dense_replace = 3,
+        norm_topk_prob = True,
+        scoring_func = 'sigmoid',
+        aux_loss_alpha = 0.001,
+        seq_aux = True,
+        hidden_act="silu",
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=0,
+        eos_token_id=1,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_nextn_predict_layers = num_nextn_predict_layers
+        self.num_attention_heads = num_attention_heads
+        self.n_shared_experts = n_shared_experts
+        self.n_routed_experts = n_routed_experts
+        self.ep_size = ep_size
+        self.routed_scaling_factor = routed_scaling_factor
+        self.kv_lora_rank = kv_lora_rank
+        self.q_lora_rank = q_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.v_head_dim = v_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.topk_method = topk_method
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.num_experts_per_tok = num_experts_per_tok
+        self.moe_layer_freq = moe_layer_freq
+        self.first_k_dense_replace = first_k_dense_replace
+        self.norm_topk_prob = norm_topk_prob
+        self.scoring_func = scoring_func
+        self.aux_loss_alpha = aux_loss_alpha
+        self.seq_aux = seq_aux
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "do_sample": true,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0653dca2ed0be0caf03a5aa3db3bcbf68a0e79d2ca8765ccd207a379c803f0e9
+size 4825751392

model-00002-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b12e9e387abadd8c61e553ce80ab6b899492a18ff4010b4807d19c0715d0fa7c
+size 4983801968

model-00003-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d770869f9adb3d3c08dc98a68055d1a7cd2810eeb095c70844689921b1cec79
+size 4991242864

model-00004-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d70d02a0e10f7f4f9544d90a54ad38d56188c7b77d23b2607b5f9e922d5b1d29
+size 4991243024

model-00005-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bfa7281afec6686c9a7969979611eb6918c5f43f04b2297a4a55927ab1ecc02
+size 4986098256

model-00006-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a473e30455d15ac7839f99e44a54922c6933b433d6f36d88cc374836cee01f43
+size 4991242880

model-00007-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ca0c9d13447f11ec1cf12e227dbbeffafb3d2ab0ff22c1c6475aa96da35a790
+size 4839298064

model-00008-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da0dd556bdc9e8b7fc24a3ff515248ef07bd8e578ebe319da0d8ba20b7201fe7
+size 4991241952

model-00009-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d476febe3c733daff48e2958af8ace6682f6aa40178d9258e765301d09d6cd3
+size 4991242888

model-00010-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693db222ebe63473dc56a94aa517e702d871f0354ce40d7a64c4acd74624866d
+size 4986098432

model-00011-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:117161b04ff34b52fb541533291690ef34900c03c91f7d0396e6bca33f80be2d
+size 4991242832

model-00012-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:947b8a3bc7c47d9001d0999690e7ae636f407b4c542ec129b27539b8117875a3
+size 4991242904

model-00013-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d77e6b5666d981cf2a480fcdf87dfefbe4ded87e2c3634df935bd5ead0b9f488
+size 4986098400

model-00014-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4116e4e3e68d22978c12d02d29d038317f6fa44e64b1139529f148d8315206a5
+size 4991242848

model-00015-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d7ad902f2dd4634ff2f61be94b923b3c7ad2bfed9a41a1fbb735e29c203d0d8
+size 4991242920

model-00016-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b7cfd4f85368556a9ec9f247f282e8564bee0e7a5608e06ee56a9098886b4fd
+size 4986098376

model-00017-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23e101bb6523fd5c2f32154a89d94e8ce4f369ae3e1287a2d79f33c4b7f05792
+size 4991242856

model-00018-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:349d591c992e0004edc6a3275774171fceebf900adc43d98477914dd310617f9
+size 4991242928

model-00019-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47fc7f1165f136551de7de116ad1921c0f8ce9007293bb88712eec55ef7c68f5
+size 4986098368

model-00020-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67e1cb0371b8dc0f2600ab064214fc554fcfedc264359c73a04dc3e21aa1ba7b
+size 4991242856

model-00021-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fba0a3a0c27f317bbf2e43449e52abaf532e775ce1913a7b855f1efeacd0709
+size 4991242944

model-00022-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5df7fbf5ca7ddf00439cb4239830f58a5d5bd74e0800d3cadec85cc31cb563f5
+size 4986098432

model-00023-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2550fd76fdcd7bd8a80d8f842cde88ae593d0f8979c127e5e1b219b14ffebd0d
+size 4991243032

model-00024-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13bffd9cdbaa43af4ae80145812deb072fa32cfc70dbe0cc946621fc15016d75
+size 4991243128

model-00025-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cfa73b613a2b0f8a08c6164d3551caf3a1497c98ddfbee0a94bba17a6be8032
+size 4986098504

model-00026-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d35733b57cdef3c6a4fe930147c189ffb0b8fc3eb312bc22664311e6e2c399c
+size 4991243024

model-00027-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3a945348444106ab6fa2b85f9c5a6fe6f87820588a37302855171fa770fa40
+size 4991243144

model-00028-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d271d12d6e70a0e0ec0ffef13ae60ed4f7b74d0f16b3d137cbfb2592c67894
+size 4986098488

model-00029-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68d5762e69416fd5967ac5601e944b53d28d9307f957381f21028f965003fd82
+size 4991243024

model-00030-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:958b205719b0c72af1afd02e441c51f7edfbbd4aa0aa00313655a4e56610e116
+size 4991243160

model-00031-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33ba538b7e8ad374b37e3a36e0b9dc5db41b5a79eb47ce4d3d90b6f00cb6c318
+size 4986098472

model-00032-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9baf7091733f62c75c876ed52980348b28cd42f914664e6e257779f8184a7a3e
+size 4991243032

model-00033-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eb3a78913a24c21b25c36a778ff67bcccc3d69844020df3dc3887a4f45e5aa7
+size 4991243168

model-00034-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:273db85f6096943bb33a51efa3dbf0fedaebec5c32d3623c48661e2b0ab876fd
+size 4986098464

model-00035-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee6ffb7c676fb48db8fccc4c1377c487ff7c483e60e679651e67684dc025dd5c
+size 4991243024

model-00036-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6566f492e34df5a1555f52063b8155565ffbbfd6a6b61ab416405e4cf247aa3
+size 4991243184

model-00037-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c5f3ddb5c31e2e525f68cfda5576e9792b8634f73d6c448aa05625b9712bb56
+size 4986098448

model-00038-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f330fc60ebd66751f7cf2e2dcf0d288c9f8c02cddef96a2da1d20728dcc2cd2
+size 4991243032

model-00039-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e3a58b7a64a9762df520f82bbba4bd04a2942f258250c79b4f2272234194fb8
+size 4991243200

model-00040-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6a1d49c7be300879805235be24dfc63f91162be9492b171706783f4c468ac06
+size 4986098432

model-00041-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f747440a56f9e0cd68f316206dc8e7ce0676d25f0115b8bb76df7f3fd7debb8
+size 4991243048

model-00042-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd1eb16be4fb04568101b8975aa11c372ca4c6af759807ba3a133ab0526d20b
+size 4927509056

model-00043-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e163db88c59aa4830f229aab6ba22c3450f4133148ac80752b9f117c5a605b84
+size 4991112056

model-00044-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2814af3a989731aff868de51dda0dbe3e08d016a42dfcbc2844136033c92b01c
+size 4991243056

model-00045-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a144688741be09a4ddd058453bd8cb492cf1adc5ae53fb364028840322d50c79
+size 4986098608