finished inference tagging pipeline

.gitignore

@@ -1,2 +1,3 @@
 .venv/
-.env
+.env
+wandb/

app.py

@@ -7,7 +7,8 @@ token_knowledge_classifier = pipeline(model="jjzha/jobbert_knowledge_extraction"
 
 examples = [
         "Knowing Python is a plus",
-        "Recommend changes, develop and implement processes to ensure compliance with IFRS standards"
+        "Recommend changes, develop and implement processes to ensure compliance with IFRS standards",
+        "Experience with Unreal and/or Unity and/or native IOS/Android 3D development and/or Web based 3D engines",
         ]
 
 

data/test-short.json

@@ -0,0 +1,20 @@
+{"idx": 1, "tokens": ["Full", "Stack", "Software", "Engineer", "-", "Java", "/", "JavaScript"], "tags_skill": ["O", "O", "O", "O", "O", "O", "O", "O"], "tags_knowledge": ["O", "O", "O", "O", "O", "O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["<ORGANIZATION>", "<ORGANIZATION>", "<ORGANIZATION>", "<ORGANIZATION>", "."], "tags_skill": ["O", "O", "O", "O", "O"], "tags_knowledge": ["O", "O", "O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["<ADDRESS>", "<ADDRESS>", "<LOCATION>", "-", "<LOCATION>"], "tags_skill": ["O", "O", "O", "O", "O"], "tags_knowledge": ["O", "O", "O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Date", "posted:", "2021-03-04"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Likes:", "0", "Dislikes:", "0", "Love:", "0"], "tags_skill": ["O", "O", "O", "O", "O", "O"], "tags_knowledge": ["O", "O", "O", "O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Salary:", "<SALARY>"], "tags_skill": ["O", "O"], "tags_knowledge": ["O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Job", "type:", "FULL_TIME"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Experience", "level:", "<EXPERIENCE>"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Industry:", "<INDUSTRY>"], "tags_skill": ["O", "O"], "tags_knowledge": ["O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Company", "size:", "<SIZE>"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Company", "type:", "<COMPANY_TYPE>"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Technologies:"], "tags_skill": ["O"], "tags_knowledge": ["O"], "source": "tech"}
+{"idx": 1, "tokens": ["javascript", "reactjs", "java"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["B", "B", "B"], "source": "tech"}
+{"idx": 1, "tokens": ["Job", "description:"], "tags_skill": ["O", "O"], "tags_knowledge": ["O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Job", "type:"], "tags_skill": ["O", "O"], "tags_knowledge": ["O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Full-time"], "tags_skill": ["O"], "tags_knowledge": ["O"], "source": "tech"}
+{"idx": 1, "tokens": ["Role:"], "tags_skill": ["O"], "tags_knowledge": ["O"], "source": "tech"}
+{"idx": 1, "tokens": ["Full", "Stack", "Developer"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["O", "O", "O"], "source": "tech"}
+{"idx": 1, "tokens": ["Technologies"], "tags_skill": ["O"], "tags_knowledge": ["O"], "source": "tech"}
+{"idx": 1, "tokens": ["javascript", "reactjs", "java"], "tags_skill": ["O", "O", "O"], "tags_knowledge": ["B", "B", "B"], "source": "tech"}

debug.py

@@ -1,40 +0,0 @@
-import spacy
-import re
-
-nlp = spacy.load("en_core_web_sm")
-
-def split_text_recursively(text):
-    if '\n' not in text:
-        return [text]
-    parts = text.split('\n', 1)
-    return [parts[0]] + split_text_recursively(parts[1])
-
-def parse_post(path):
-
-    # Read the file
-
-    with open(path, 'r') as file:
-        text = file.read()
-
-    # Sentence tokenization
-
-    str_list = split_text_recursively(text)
-    str_list = [i.strip() for i in str_list]
-    str_list = list(filter(None, str_list))
-
-    count = 0
-    sents = []
-
-    for line in str_list:
-        doc = nlp(line)
-        for sent in doc.sents:
-            print(f"{sent.text}")
-            sents.append(sent.text)
-
-    # Skill/knowledge extraction
-    
-    
-
-
-path = './job-postings/03-01-2024/2.txt'
-parse_post(path)

debug2.py

@@ -1 +0,0 @@
-deb

demo-app.py

@@ -1,56 +0,0 @@
-import gradio as gr
-from transformers import pipeline
-
-token_skill_classifier = pipeline(model="jjzha/jobbert_skill_extraction", aggregation_strategy="first")
-token_knowledge_classifier = pipeline(model="jjzha/jobbert_knowledge_extraction", aggregation_strategy="first")
-
-
-examples = [
-        "Knowing Python is a plus",
-        "Recommend changes, develop and implement processes to ensure compliance with IFRS standards"
-        ]
-
-
-def aggregate_span(results):
-    new_results = []
-    current_result = results[0]
-
-    for result in results[1:]:
-        if result["start"] == current_result["end"] + 1:
-            current_result["word"] += " " + result["word"]
-            current_result["end"] = result["end"]
-        else:
-            new_results.append(current_result)
-            current_result = result
-
-    new_results.append(current_result)
-
-    return new_results
-
-def ner(text):
-    output_skills = token_skill_classifier(text)
-    for result in output_skills:
-        if result.get("entity_group"):
-            result["entity"] = "Skill"
-            del result["entity_group"]
-
-    output_knowledge = token_knowledge_classifier(text)
-    for result in output_knowledge:
-        if result.get("entity_group"):
-            result["entity"] = "Knowledge"
-            del result["entity_group"]
-
-    if len(output_skills) > 0:
-        output_skills = aggregate_span(output_skills)
-    if len(output_knowledge) > 0:
-        output_knowledge = aggregate_span(output_knowledge)
-
-    return {"text": text, "entities": output_skills}, {"text": text, "entities": output_knowledge}
-
-
-demo = gr.Interface(fn=ner,
-                    inputs=gr.Textbox(placeholder="Enter sentence here..."),
-                    outputs=["highlight", "highlight"],
-                    examples=examples)
-
-demo.launch()

env-template.txt

@@ -0,0 +1,3 @@
+OPENAI_API_KEY=<openai api key>
+HF_USERNAME=<hugging face username>
+WANDB_API_KEY=<weights & biases api key>

examples.py → few-shot-extract.py

@@ -1,4 +1,6 @@
 import requests
+import os
+repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 def show_examples(n = 10):
 
@@ -13,11 +15,12 @@ def show_examples(n = 10):
             tokens = row['tokens']
             skill_labels, knowledge_labels = row['tags_skill'], row['tags_knowledge']
 
-            print(f'Example #{i+1}')
-            print('Tokens:', tokens)
-            print('Skill Labels:', skill_labels)
-            print('Knowledge Labels:', knowledge_labels)
-            print('')
+            with open(f"{repo_dir}/examples.txt", 'w') as file:
+                file.write(f'Example #{i+1}\n')
+                file.write(f'Tokens: {str(tokens)}\n')
+                file.write(f'Skill Labels: {str(skill_labels)}\n')
+                file.write(f'Knowledge Labels: {str(knowledge_labels)}\n')
+                file.write('\n')
 
 
 show_examples(n=100)

job-ad.txt

@@ -1,40 +0,0 @@
-About the job
-Grow with us
-
-About This Opportunity
-
-Ericsson is a world-leading provider of telecommunications equipment and services to mobile and fixed network operators. Over 1,000 networks in more than 180 countries use Ericsson equipment, and more than 40 percent of the world's mobile traffic passes through Ericsson networks. Using innovation to empower people, business and society, Ericsson is working towards the Networked Society: a world connected in real time that will open opportunities to create freedom, transform society and drive solutions to some of our planet’s greatest challenges.
-
-Ericsson's 6G vision, first introduced in 2020, remains pivotal for transforming business and society in the 2030s through secure, efficient, and sustainable communication services. As 6G development progresses into a more concrete phase of regulation and standardization we are looking for researchers that would like to join us, co-creating a cyber-physical world
-
-Within Ericsson, Ericsson Research develops new communication solutions and standards which have made Ericsson the industry leader in defining five generations of mobile communication. As we gear up for the 6th generation, we would like to fully embrace and utilize cloud native principles, hyperscalers and internal cloud infrastructure in our research. We are now looking for a MLOps research engineer to develop and support our workflows.
-
-In this role, you will
-
-Contribute to the direction and implementation of ML-based ways of working
-Study, design and develop workflows and solutions for AI based R&D
-Work across internal compute and external cloud platforms
-Working closely with researchers driving 6G standardization
-
-Join our Team
-
-Qualifications
-
-MSc in Data Science or related field, or have equivalent practical experience
-Technical skills and/or professional experience, particularly in:
-Programming in various languages (Python, Go, etc)
-MLOps technologies and tooling (e.g. MLFlow, Kubeflow)
-Dispatching and computational Python packages (Hydra, numpy, TensorFlow, etc.)
-DevOps and CI/CD experience, runner deployment & management, pipeline creation, testing etc. for validating ML-driven code
-Familiarity in the following is a plus:
-ML frameworks (PyTorch, TensorFlow, or Jax)
-Containers technologies (engines, orchestration tools and frameworks such as Docker, Kaniko, Kubernetes, Helm, etc.)
-Cloud ecosystems along with the respective infrastructure, in particular AWS
-Infrastructure management (Ansible, Terraform, etc.)
-Team skills is a necessity. Daily cross-functional collaboration and interaction with other skilled researchers are the basis for our ways of working.
-You should enjoy working with people having diverse backgrounds and competences. 
-It is important that you have strong personal drive and a strong focus on the tasks at hand.
-Ability to translate high-level objectives into detailed tasks and actionable steps.
-Location: Luleå, Sweden
-
-

tag-posting.py

@@ -1,7 +1,12 @@
 import spacy
 import re
+from transformers import AutoTokenizer, BertForTokenClassification, TrainingArguments, Trainer
+import torch
+from typing import List
+import os
 
-nlp = spacy.load("en_core_web_sm")
+
+### Parsing job posting
 
 def split_text_recursively(text):
     if '\n' not in text:
@@ -11,6 +16,8 @@ def split_text_recursively(text):
 
 def parse_post(path):
 
+    nlp = spacy.load("en_core_web_sm")
+
     # Read the file
 
     with open(path, 'r') as file:
@@ -30,11 +37,191 @@ def parse_post(path):
         for sent in doc.sents:
             print(f"{sent.text}")
             sents.append(sent.text)
+    
+    return sents
+
+
+### Model inference
+
+from torch.utils.data import DataLoader
+import torch.nn as nn
+from transformers import DataCollatorForTokenClassification
+from typing import List, Tuple
+
+tokenizer = AutoTokenizer.from_pretrained("jjzha/jobbert_knowledge_extraction")
+model = BertForTokenClassification.from_pretrained("Robzy/jobbert_knowledge_extraction")
+
+id2label = model.config.id2label
+label2id = model.config.label2id
+
+def pad(list_of_lists, pad_value=0):
+    max_len = max(len(lst) for lst in list_of_lists)
+
+    # Pad shorter lists with the specified value
+    padded_lists = [lst + [pad_value] * (max_len - len(lst)) for lst in list_of_lists]
+    attention_masks = [[1] * len(lst) + [0] * (max_len - len(lst)) for lst in list_of_lists]
+    
+    return torch.tensor(padded_lists), torch.tensor(attention_masks)
+
+def collate_fn(batch: List[List[torch.Tensor]]):
+
+    input_ids, attention_mask = pad(list(map(lambda x: tokenizer.convert_tokens_to_ids(x['tokens']),batch)))
+    tags_knowledge, _ = pad([list(map(lambda x: label2id[x],o)) for o in [b['tags_knowledge'] for b in batch]])
+    return {"input_ids": input_ids, "tags_knowledge": tags_knowledge, "attention_mask": attention_mask}
 
-    # Skill/knowledge extraction
+def extract_spans(B_mask, I_mask, token_ids, tokenizer):
+    """
+    Extract text spans for 2D tensors (batch of sequences).
+    """
+    batch_size = B_mask.size(0)
+    all_spans = []
+
+    d = tokenizer.decode
+
+    for batch_idx in range(batch_size):
+        spans = []
+        current_span = []
+
+        for i in range(B_mask.size(1)):  # Iterate over sequence length
+            if B_mask[batch_idx, i].item() == 1:  # Begin a new span
+                if current_span:
+                    spans.append(current_span)
+                    print(d(current_span))
+                current_span = [token_ids[batch_idx, i].item()]
+                print(d(current_span))
+            elif I_mask[batch_idx, i].item() == 1 and current_span:  # Continue the current span
+                print(d(current_span))
+                current_span.append(token_ids[batch_idx, i].item())
+            else:  # Outside any entity
+                print(d(current_span))
+                if current_span:
+                    spans.append(current_span)
+                    current_span = []
+
+        if current_span:  # Save the last span if it exists
+            spans.append(current_span)
+
+        # Decode spans for this sequence
+        decoded_spans = [tokenizer.decode(span, skip_special_tokens=True) for span in spans]
+        all_spans.append(decoded_spans)
+
+    # Remove empty spans
+    all_spans = list(filter(lambda x: x != [], all_spans))
+
+    return all_spans
+
+
+def concat_subtokens(tokens):
+    result = []
     
+    for token in tokens:
+        if token.startswith('##'):
+            # Concatenate sub-token to the last token in result
+            result[-1] += token[2:]  # Remove '##' and append the continuation
+        else:
+            # If it's a new token, add it to result
+            result.append(token)
     
+    return result
+
+def merge_spans(batch_spans, tokenizer):
+
+    batch_decoded_spans = []
+
+    for spans in batch_spans:
+
+        ## Concatenate subtokens
+
+        if spans[0].startswith('##'):
+            continue
+
+        decoded_spans = []
+        for token in spans:
+            if token.startswith('##'):
+                # Concatenate sub-token to the last token in result
+                decoded_spans[-1] += token[2:]  # Remove '##' and append the continuation
+            else:
+                # If it's a new token, add it to result
+                decoded_spans.append(token)
+
+        ## Concatenatation done
+ 
+        for span in decoded_spans:
+            batch_decoded_spans.append(span)
+
+    return batch_decoded_spans
+
+
+def extract_skills(batch_sentences: List[str]):
+
+    print('Extracting skills from job posting...')
+
+    # Batch
+
+    # Tokenize
+    batch = tokenizer(batch_sentences, padding=True, truncation=True) 
+    batch_tokens = torch.tensor(batch['input_ids'])
+    batch_attention_masks = torch.tensor(batch['attention_mask'])
+
+    model.eval()
+    with torch.no_grad():
+        output = model(input_ids=batch_tokens, attention_mask=batch_attention_masks)
+
+    # Post-process
+    pred = output.logits.argmax(-1)
+    pred = torch.where(batch_attention_masks==0, torch.tensor(-100), pred)
+
+    b_mask = torch.where(pred==0, 1, 0)
+    i_mask = torch.where(pred==1, 1, 0)
+
+    spans = extract_spans(b_mask, i_mask, batch_tokens, tokenizer)
+    decoded_spans = merge_spans(spans, tokenizer)
+
+    return decoded_spans
+
+def skills_save(path,skills):
+    with open(path, 'w') as f:
+        for i, skill in enumerate(skills):
+            if i == len(skills) - 1:
+                f.write(f"{skill}")
+            else:
+                f.write(f"{skill}\n")
+
+
+def backfill():
+
+    job_dir = os.path.join(os.getcwd(), 'job-postings')
+    tag_dir = os.path.join(os.getcwd(), 'tags')
+
+    for date in os.listdir(job_dir):
+        print(f"Processing date directory: {date}")
+        
+        job_date = os.path.join(job_dir, date)
+        tag_date = os.path.join(tag_dir, date)
+
+        for job in os.listdir(job_date):
+            job_path = os.path.join(job_date, job)
+            tag_path = os.path.join(tag_date, job)
+
+            print(f"Processing job file: {job_path}")
+
+            if not os.path.exists(tag_date):
+                os.makedirs(tag_date)
+                print(f"Created directory: {tag_date}")
+
+            sents = parse_post(job_path)
+            skills = extract_skills(sents)
+            skills_save(tag_path, skills)
+
+            print(f"Saved skills to: {tag_path}")
+
+if __name__ == '__main__':
+
+    # Backfill
+    backfill()
 
 
-path = './job-postings/03-01-2024/2.txt'
-parse_post(path)
+    # path = './job-postings/03-01-2024/2.txt'
+    # sents = parse_post(path)
+    # skills = extract_skills(sents)
+    # skills_save('./tags/03-01-2024/2.txt',skills)

tags/03-01-2024/1.txt

@@ -1 +1,34 @@
-tags
+ML
+-
+AI based R & D
+MSc in Data Science
+Python
+Go
+MLOps
+MLFlow
+Kubeflow )
+Hydra
+numpy
+TensorFlow
+DevOps
+CI
+/
+CD
+runner deployment & management
+pipeline creation
+testing
+ML
+ML
+PyTorch
+TensorFlow
+Containers
+engines, orchestration tools and
+Docker
+Kaniko
+Kubernetes
+Helm
+Cloud ecosystems
+AWS
+Infrastructure management
+Ansible
+Terraform

tags/03-01-2024/2.txt

@@ -1 +1,13 @@
-tags
+artificial intelligence
+Automation
+data analysis
+image recognition
+automation
+Artificial Intelligence
+feasibility studies
+data analysis
+Data Science
+degree in software engineering
+Artificial Intelligence
+Vision Systems
+English

tags/03-01-2024/3.txt

@@ -1 +1,22 @@
-tags
+SQL
+cloud infrastructure
+APIs
+Python
+infra
+database
+Types
+SaaS
+agile development
+sprint planning
+backend development
+python
+SQL
+NoSQL databases
+web scraping
+API development
+containerization
+cloud environments
+Azure
+data processing
+Databricks
+English

tags/04-01-2024/1.txt

@@ -1 +1,36 @@
-tags
+Defence projects
+machine learning
+artificial intelligence
+AI models
+AI systems
+AI
+Master
+'
+s or Ph. D. in Computer Science
+Machine Learning
+Pattern Recognition
+Neural Networks
+Algorithms
+AI
+/
+ML
+autonomous systems
+radar technologies
+AI
+-
+reliant
+defense
+machine learning frameworks
+TensorFlow
+PyTorch
+Python
+,
+C
++
++
+Java
+secure system design
+cybersecurity principles
+Security certifications
+CISSP
+CEH )

tags/04-01-2024/2.txt

@@ -1 +1,36 @@
-tags
+Spatial Computing /
+XR Development
+game
+Swedish
+real
+3D graphics
+Real Time Graphics
+VR
+/
+MR
+/
+AR )
+graphics pipelines
+real
+-
+time 3D environments
+Unreal
+Unity
+native
+IOS
+/
+Android 3D development
+Web based 3D engines
+mobile application development
+deployment
+game
+3D Graphics
+C
+,
+C
+#
+Python
+C
++
++
+JavaScript

tags/04-01-2024/3.txt

@@ -1 +1,44 @@
-tags
+machine
+AI
+SaaS
+AI
+/
+ML
+AI
+/
+ML models
+AI
+AI
+/
+ML pipelines
+deployment infrastructure
+Python
+AI
+/
+ML
+Pytorch
+cloud environment
+Azure
+AWS
+GCP
+AI
+Master
+'
+s degree in engineering
+Cloud Ops
+IaC
+Terraform
+MLOps best practices and tools
+Databricks
+VRDs
+)
+generative AI
+RAG
+LLM evaluation
+API
+-
+driven microservices
+cache management
+production
+-
+level software

tags/07-01-2025/1.txt

@@ -0,0 +1,53 @@
+commodity recommendations
+live stream recommendations
+short video recommendations
+TikTok
+feature engineering
+model optimization
+Master
+'
+s degree
+Phd
+'
+s Degree
+Software Development
+Computer Science
+Computer Engineering
+machine learning
+deep learning
+data mining
+programming language
+C
++
++
+/
+Python
+Deep Learning Tools
+tensorflow
+/
+pytorch
+Collaborative Filtering
+Matrix Factorization
+Factorization Machines
+Word2vec
+Logistic Regression
+Gradient Boosting
+Trees
+Deep Neural Networks
+Wide and Deep
+KDD
+NeurlPS
+WWW
+SIGIR
+WSDM
+ICML
+IJCAI
+AAAI
+RECSYS
+data mining
+machine learning
+Kaggle
+/
+KDD
+-
+cup

tags/07-01-2025/10.txt

@@ -0,0 +1,44 @@
+feature development
+Data Drivens
+machine learning
+algorithm development
+model training
+feature pipeline design
+A
+/
+B testing
+Python
+machine learning algorithms and workflows
+NLP
+Deep Learning
+Recommendation Systems
+Conversational
+English
+recommendation systems
+search
+e
+-
+commerce
+advertising
+NLP
+Chinese text analysis
+business applications
+system design
+machine learning systems
+ML
+Scikit
+-
+Learn
+/
+XGBoost
+/
+Tensorflow
+GCP
+/
+Kubernetes
+SQL
+/
+NoSQL
+/
+Redis
+Linux

tags/07-01-2025/2.txt

@@ -0,0 +1,96 @@
+Deep Learning
+MLOps
+production environments
+model management
+automation
+continuous integration
+deep
+MLOps
+Deep
+CNNs
+RNNs
+Transformers
+NLP
+computer vision
+predictive analytics
+MLOps
+Pipeline Development
+M
+model training
+Model De
+CI
+/
+CD
+model versioning
+lifecycle management
+Kubernetes
+Docker
+cloud platforms
+AWS
+,
+Azure
+GCP
+cloud platforms
+AWS SageMaker
+Google AI Platform
+Azure
+Machine Learning
+Cross
+-
+Functional Collaboration
+machine learning
+deep learning
+MLOps
+TensorFlow
+Keras
+PyTorch
+MLOps
+Kubeflow
+MLflow
+TFX
+Jenkins
+Docker
+Kubernetes
+Terraform
+Python
+data manipulation libraries
+Pandas
+NumPy
+SciPy
+cloud platforms
+AWS
+GCP
+Azure
+machine learning
+AWS
+SageMaker
+Google AI Platform
+Azure
+ML
+NLP
+computer vision
+reinforcement learning
+MLOps
+open
+-
+source
+MLOps
+Kubeflow
+MLflow
+TFX
+end
+machine learning lifecycle
+infrastructure as code tools
+Terraform
+CloudFormation
+MLOps
+Continuous Learning
+deep learning
+MLOps practices
+model deployment strategies
+Master
+'
+s or PhD in
+Computer Science
+Data Science
+Electrical Engineering

tags/07-01-2025/3.txt

@@ -0,0 +1,38 @@
+PhD degree in Computer Science
+Python
+JavaScript
+R
+Java
+C
++
++
+Machine Learning
+Python
+JavaScript
+R
+Java
+C
++
++
+automated algorithm discovery methods
+learning to learn
+program synthesis
+digital hardware
+machine learning
+computational neuroscience
+non
+-
+gradient
+-
+based optimization techniques
+hand
+-
+automated discovery
+machine learning
+modern programming languages
+Python
+computation methods
+machine learning libraries
+JAX
+PyTorch
+)

tags/07-01-2025/4.txt

@@ -0,0 +1,38 @@
+PhD degree in Computer Science
+Python
+JavaScript
+R
+Java
+C
++
++
+Machine Learning
+Python
+JavaScript
+R
+Java
+C
++
++
+automated algorithm discovery methods
+learning to learn
+program synthesis
+digital hardware
+machine learning
+computational neuroscience
+non
+-
+gradient
+-
+based optimization techniques
+hand
+-
+automated discovery
+machine learning
+modern programming languages
+Python
+computation methods
+machine learning libraries
+JAX
+PyTorch
+)

tags/07-01-2025/5.txt

@@ -0,0 +1,38 @@
+PhD degree in Computer Science
+Python
+JavaScript
+R
+Java
+C
++
++
+Machine Learning
+Python
+JavaScript
+R
+Java
+C
++
++
+automated algorithm discovery methods
+learning to learn
+program synthesis
+digital hardware
+machine learning
+computational neuroscience
+non
+-
+gradient
+-
+based optimization techniques
+hand
+-
+automated discovery
+machine learning
+modern programming languages
+Python
+computation methods
+machine learning libraries
+JAX
+PyTorch
+)

tags/07-01-2025/6.txt

@@ -0,0 +1,39 @@
+AI
+Large Language Models ( LLMs )
+Generative AI algorithms
+neural networks
+ML
+PyTorch
+TensorFlowL
+complex
+IP
+computer science
+software engineering
+TensorFlow
+PyTorch
+Python
+Large Language Models ( LLMs )
+Generative AI algorithms
+software development platforms
+continuous integration systems
+Linux and cloud services
+Pytorch
+Tensorflow
+Executorch
+Tensorflow Lite
+CI
+/
+testing
+Python
+ML
+C
++
++
+optimised
+ML libraries
+machine learning
+machine learning models
+proof -
+ARM IPs
+machine
+ML

tags/07-01-2025/7.txt

@@ -0,0 +1,77 @@
+continuous learning
+modern data science
+analytics
+software engineering
+academic degrees
+Bachelor
+'
+s degree
+AI
+Machine Learning
+Python
+Generative AI models
+OpenAI family
+open source
+LLMs
+Dall
+-
+e
+LlamaIndex
+Langchain
+Retrieval
+Augmented Generation
+RAG )
+ML
+scikit
+-
+learn
+Pytorch
+ONNX
+ML
+DevOps
+GIT
+Azure Devops
+Agile
+Jira
+Machine Learning
+ML ) workflows
+MLOps
+MLFlow
+CI
+/
+CD
+test
+-
+driven development
+ML models
+ML
+data structures
+data modelling
+software engineering best practices
+data manipulation
+SQL
+Pandas
+Spark
+containerization
+scaling models
+AI
+calculus
+linear algebra
+statistics
+Master
+'
+s degree
+Computer Science
+Mathematics
+Physical Sciences
+Python
+R
+JavaScript
+Java
+,
+C
++
++
+C
+Generative AI models
+ale

tags/07-01-2025/8.txt

@@ -0,0 +1,45 @@
+convolutional
+neural networks
+autoencoders
+transformer models
+digital pathology
+single cell transcriptomics
+H
+E
+transfer learning
+shallow machine learning
+H
+&
+E images
+single cell transcriptomics
+multi
+-
+modal
+single cell transcriptomics
+medical images
+tumor microenvironment
+drug discovery & development
+AI
+/
+ML
+Chemistry
+/
+Biology
+/
+Biochemistry
+MS Office
+PowerPoint
+Words
+Excel
+e
+-
+mails
+group messaging
+information gathering
+quantitative
+bioinformatics
+biomedical engineering
+machine learning
+math
+statistics
+real projects

tags/07-01-2025/9.txt

@@ -0,0 +1,5 @@
+ML
+LLMs
+RL
+open source
+machine learning

train.py

@@ -0,0 +1,178 @@
+from transformers import AutoTokenizer, BertForTokenClassification, TrainingArguments, Trainer
+import torch
+from tabulate import tabulate
+import wandb
+
+
+tokenizer = AutoTokenizer.from_pretrained("jjzha/jobbert_knowledge_extraction")
+model = BertForTokenClassification.from_pretrained("Robzy/jobbert_knowledge_extraction")
+
+artifact = wandb.Artifact(name="jobbert-knowledge-extraction", type="BERT")
+
+text = 'Experience with Unreal and/or Unity and/or native IOS/Android 3D development and/or Web based 3D engines '
+
+# Tokenize  
+inputs = tokenizer(
+    text, add_special_tokens=False, return_tensors="pt"
+)
+
+# Inference
+
+# with torch.no_grad():
+#     output = model(**inputs)
+
+# # Post-process
+# predicted_token_class_ids = output.logits.argmax(-1)
+# predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
+# tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'].squeeze())
+
+# # Display
+# table = zip(tokens, predicted_tokens_classes)
+# print(tabulate(table, headers=["Token", "Predicted Class"], tablefmt="pretty"))
+
+# Training
+
+from datasets import load_dataset
+dataset = load_dataset("json", data_files="data/test-short.json")
+
+
+# Convert tokens to ids before training
+
+data = [torch.tensor([tokenizer.convert_tokens_to_ids(t) for t in l]) for l in dataset['train']['tokens']]
+
+dataset = dataset.map(
+    lambda x: {"input_ids": torch.tensor(tokenizer.convert_tokens_to_ids(x["tokens"]))}
+)
+
+# Data preprocessing
+
+from torch.utils.data import DataLoader
+import torch.nn as nn
+from transformers import DataCollatorForTokenClassification
+from typing import List, Tuple
+
+def pad(list_of_lists, pad_value=0):
+    max_len = max(len(lst) for lst in list_of_lists)
+
+    # Pad shorter lists with the specified value
+    padded_lists = [lst + [pad_value] * (max_len - len(lst)) for lst in list_of_lists]
+    attention_masks = [[1] * len(lst) + [0] * (max_len - len(lst)) for lst in list_of_lists]
+    
+    return torch.tensor(padded_lists), torch.tensor(attention_masks)
+
+
+def collate_fn(batch: List[List[torch.Tensor]]):
+
+    input_ids, attention_mask = pad(list(map(lambda x: tokenizer.convert_tokens_to_ids(x['tokens']),batch)))
+    tags_knowledge, _ = pad([list(map(lambda x: label2id[x],o)) for o in [b['tags_knowledge'] for b in batch]])
+    return {"input_ids": input_ids, "tags_knowledge": tags_knowledge, "attention_mask": attention_mask}
+
+# Training settings
+batch_size = 32
+train_dataloader = DataLoader(dataset['train'], shuffle=True, batch_size=batch_size, collate_fn=collate_fn)
+eval_dataloader = DataLoader(dataset['train'], batch_size=batch_size, collate_fn=collate_fn)
+
+from tqdm.auto import tqdm
+from torch.optim import AdamW
+from transformers import get_scheduler
+
+model.train()
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+IGNORE_INDEX = -100
+criterion = nn.CrossEntropyLoss(ignore_index=IGNORE_INDEX)
+id2label = model.config.id2label
+label2id = model.config.label2id
+
+lr = 5e-5
+optimizer = AdamW(model.parameters(), lr=lr)
+
+num_epochs = 3
+num_training_steps = num_epochs * len(train_dataloader)
+lr_scheduler = get_scheduler(
+    name="linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps
+)
+
+model.config.pad_token_id = 0
+
+## Training
+
+from dotenv import load_dotenv
+import os
+load_dotenv(".env")
+
+from datetime import datetime
+current_time = datetime.now()
+
+wandb.login(key=os.getenv('WANDB_API_KEY'))
+
+run = wandb.init(
+    # set the wandb project where this run will be logged
+    project="in-demand",
+
+    # track hyperparameters and run metadata
+    config={
+    "learning_rate": lr,
+    "architecture": "BERT",
+    "epochs": num_epochs,
+    "batch_size": batch_size,
+    "notes": "Datetime: " + current_time.strftime("%m/%d/%Y, %H:%M:%S")
+    }
+)
+
+import logging
+from datetime import datetime
+logging.info("Initiating training")
+
+progress_bar = tqdm(range(num_epochs), desc="Epochs")
+for epoch in range(num_epochs):
+    logging.info(f"Epoch #{epoch}")
+    print(f"Epoch #{epoch}")
+
+    batch_count = 0
+
+    for batch in train_dataloader:
+
+        logging.info(f"Batch #{batch_count} / {len(train_dataloader)}")
+        print(f"Batch #{batch_count} / {len(train_dataloader)}")
+
+        tokens = batch['input_ids'].to(device)
+        attention_mask = batch['attention_mask'].to(device)
+        tags_knowledge = batch['tags_knowledge'].to(device)
+
+        outputs = model(tokens, attention_mask=attention_mask)
+
+        # Batch
+        pred = outputs.logits.reshape(-1, model.config.num_labels) # Logits
+        label = torch.where(attention_mask==0, torch.tensor(IGNORE_INDEX).to(device), tags_knowledge).reshape(-1) # Labels, padding set to class idx -100
+
+        # Compute accuracy ignoring padding idx
+        _, predicted_labels = torch.max(pred, dim=1)
+        non_pad_elements = label != IGNORE_INDEX
+        correct_predictions = (predicted_labels[non_pad_elements] == label[non_pad_elements]).sum().item()
+        total_predictions = non_pad_elements.sum().item()
+        accuracy = correct_predictions / total_predictions if total_predictions > 0 else 0
+
+        loss = criterion(pred, label)
+        loss.backward()
+        optimizer.step()
+        lr_scheduler.step()
+        optimizer.zero_grad()
+        
+        wandb.log({"epoch": epoch, "accuracy": accuracy, "loss": loss})
+
+        batch_count += 1
+
+    progress_bar.update(1)
+
+
+model.push_to_hub("Robzy/jobbert_knowledge_extraction")
+
+
+# Add the state_dict to the artifact
+state_dict = model.state_dict()
+with artifact.new_file('model.pth', mode='wb') as f:
+    torch.save(state_dict, f)
+
+# Log the artifact to W&B
+wandb.log_artifact(artifact)