changes

debug.py

@@ -0,0 +1,40 @@
+import spacy
+import re
+
+nlp = spacy.load("en_core_web_sm")
+
+def split_text_recursively(text):
+    if '\n' not in text:
+        return [text]
+    parts = text.split('\n', 1)
+    return [parts[0]] + split_text_recursively(parts[1])
+
+def parse_post(path):
+
+    # Read the file
+
+    with open(path, 'r') as file:
+        text = file.read()
+
+    # Sentence tokenization
+
+    str_list = split_text_recursively(text)
+    str_list = [i.strip() for i in str_list]
+    str_list = list(filter(None, str_list))
+
+    count = 0
+    sents = []
+
+    for line in str_list:
+        doc = nlp(line)
+        for sent in doc.sents:
+            print(f"{sent.text}")
+            sents.append(sent.text)
+
+    # Skill/knowledge extraction
+    
+    
+
+
+path = './job-postings/03-01-2024/2.txt'
+parse_post(path)

debug2.py

@@ -0,0 +1 @@
+deb

job-postings/03-01-2024/1.txt

@@ -0,0 +1,40 @@
+ About the job
+
+Grow with us
+
+About This Opportunity
+
+Ericsson is a world-leading provider of telecommunications equipment and services to mobile and fixed network operators. Over 1,000 networks in more than 180 countries use Ericsson equipment, and more than 40 percent of the world's mobile traffic passes through Ericsson networks. Using innovation to empower people, business and society, Ericsson is working towards the Networked Society: a world connected in real time that will open opportunities to create freedom, transform society and drive solutions to some of our planet’s greatest challenges.
+
+Ericsson's 6G vision, first introduced in 2020, remains pivotal for transforming business and society in the 2030s through secure, efficient, and sustainable communication services. As 6G development progresses into a more concrete phase of regulation and standardization we are looking for researchers that would like to join us, co-creating a cyber-physical world
+
+Within Ericsson, Ericsson Research develops new communication solutions and standards which have made Ericsson the industry leader in defining five generations of mobile communication. As we gear up for the 6th generation, we would like to fully embrace and utilize cloud native principles, hyperscalers and internal cloud infrastructure in our research. We are now looking for a MLOps research engineer to develop and support our workflows.
+
+In this role, you will
+
+    Contribute to the direction and implementation of ML-based ways of working
+    Study, design and develop workflows and solutions for AI based R&D
+    Work across internal compute and external cloud platforms
+    Working closely with researchers driving 6G standardization
+
+Join our Team
+
+Qualifications
+
+    MSc in Data Science or related field, or have equivalent practical experience
+    Technical skills and/or professional experience, particularly in:
+        Programming in various languages (Python, Go, etc)
+        MLOps technologies and tooling (e.g. MLFlow, Kubeflow)
+        Dispatching and computational Python packages (Hydra, numpy, TensorFlow, etc.)
+        DevOps and CI/CD experience, runner deployment & management, pipeline creation, testing etc. for validating ML-driven code
+    Familiarity in the following is a plus:
+        ML frameworks (PyTorch, TensorFlow, or Jax)
+        Containers technologies (engines, orchestration tools and frameworks such as Docker, Kaniko, Kubernetes, Helm, etc.)
+        Cloud ecosystems along with the respective infrastructure, in particular AWS
+        Infrastructure management (Ansible, Terraform, etc.)
+    Team skills is a necessity. Daily cross-functional collaboration and interaction with other skilled researchers are the basis for our ways of working.
+    You should enjoy working with people having diverse backgrounds and competences. 
+    It is important that you have strong personal drive and a strong focus on the tasks at hand.
+    Ability to translate high-level objectives into detailed tasks and actionable steps.
+
+Location: Luleå, Sweden

job-postings/03-01-2024/2.txt

@@ -0,0 +1,51 @@
+
+About the job
+
+Share this job
+
+         
+
+For our research & development team at Transformers we are now looking for an R&D Engineer with expertise in the area of artificial intelligence and Automation. You will join our team to develop smart solutions for future generations of power transformers and transformer components. Your focus will be technical contribution towards data analysis, image recognition, automation solution, investigation, and coordination of improvements within Transformers. You will also have ability to work with cross-competence topics and contribute to development of new and innovative solutions in different areas.
+
+You will work in dynamic, motivated, and creative team with a wide range of experience and competence. Through your research projects you will become part of global community that will shape and help you to develop your career into a technical or managerial leader!
+
+Your Responsibilities
+
+    Run, within the Artificial Intelligence field, feasibility studies, propose, and develop new technical concepts, design, and implement new solutions.
+    Integrating AI projects within our production leading the way towards the industry 4.0 revolution.
+    Organize prototyping and testing activities to validate new technologies/algorithms.
+    Provide technical support to factories in the field of operational excellence and improving the processes as well as data analysis of different stages of the production.
+    Have global collaboration with scientists of Hitachi energy’s research teams and technology centers, technology managers and engineers from the various factories.
+    Be an active member in further strengthening the collaboration with universities and research institutes.
+
+Your background
+
+    You have experience working in the field of Data Science and a degree in software engineering or equivalent.
+    You also have experience in integration of a system with different existing hardware as well as building new setups from scratch. Experience with Artificial Intelligence Vision Systems is a plus.
+    You have sound analytical skills and have ambition towards applying new and innovative solutions to improve the current solutions.
+    You are a team player who enjoys collaborating with others to achieve great results.
+    Preferably you also have experience in, or ability to develop, project management and technical leadership skills.
+    You have good communication skills and are fluent in English, both written and spoken
+
+What We Offer
+
+    Collective agreement 
+    Flexible working time 
+    Health care and wellness allowance 
+    Fantastic career possibilities within Hitachi Energy both within Sweden and globally 
+    Mentor to support you throughout onboard phase 
+    Various trainings and education supporting employee development 
+    Diversified company with over 70+ nationalities working in Sweden 
+    Supplementary compensation for parental leave 
+    Employee Benefit Portal with thousands of discounts and perks 
+    More benefits could be connected to this specific role
+
+Additional Information
+
+Are you ready for an exciting new challenge? Does the above description sound like you? Applications will be reviewed on an ongoing basis, so don’t delay – apply today!
+
+Recruiting Manager Nima Sadr-Momtazi, [email protected], will answer your questions about the position.  Union representatives - Sveriges Ingenjörer: Håkan Blomquist, +46 107-38 31 52; Unionen: Tomas L. Gustafsson, +46 107-38 27 47; Ledarna: Frank Hollstedt, +46 107-38 70 43. All other questions can be directed to Recruiter Gustaf Hedström, [email protected].
+
+ Apply now 
+
+Location Ludvika, Dalarna County, Sweden Job type Full time Experience Experienced Job function Engineering & Science Contract Regular Publication date 2024-10-01 Reference number R0052858

job-postings/03-01-2024/3.txt

@@ -0,0 +1,60 @@
+ About the job
+
+Tembi is looking for two senior python developers to help us develop our data collection engine and web scraping tech. Work with SQL, cloud infrastructure, APIs and large volumes of data. We have just secured a €3 million funding round having full focus on taking our Market Intelligence Platform to the next level.
+
+
+Tembi's Market Intelligence platform connects data from multiple industries and uses advanced analytics to create tailored market insights and predictions for various industries. From relocation predictions to i.e. company growth projections, Tembi provides users with actionable market analyses to discover new growth and sales opportunities. With Tembi, B2B companies gain a better overview and understanding of their market's development. They use the data and insights to make more informed strategic decisions, and execute on them with Tembi's lead generation solution.
+
+
+Our platform is developed by an internal team that poses all the competencies involved in building data driven SaaS web products. We have data engineers, data scientists, data analysts, UI/UX designers, front end developers and agile coaches. We are committed to our company values to innovate, collaborate and support each other. We celebrate our wins and take every opportunity there is to improve and to learn new things. We believe in agile ways of working and apply a start-up mentality in everything we do.
+
+
+Location: The role is located in Copenhagen, Denmark. Our office is located 5 min walk from Nørreport station.
+
+
+Who are you?
+
+We are looking for a highly skilled senior backend developer with experience in Python along with infra and database experience. We utilize large-scale scraping technologies built in Python, hosted and being run on containerized environments in the cloud. Data is processed and served to our frontend via APIs written in Typescript running on a node server.
+
+
+You will be a key-player in development and optimization of our scraping technologies, making sure they run smoothly, and data is served to our frontend and the customers. You will work very closely with the rest of the team in optimizing data for both analytics & our SaaS application platform.
+
+
+On a day-to-day basis you will be a key player in our agile development process, and part of sprint planning, team retrospectives and setting the right product vision and road map. We want to take new ideas to our customers quickly and measure the outcome to make sure we create the right value. Your input on delivering fast while safeguarding quality will be a priority and how we will evaluate your success.
+
+
+On a personal side, leadership skills, growth mindset and a get-things-done mentality is something we value a lot. Things move fast and we make adoptions all the time in favor of reaching our goals. There will be many opportunities for the right person to grow, try out new things and be part of a great team with a unicorn vision.
+
+
+Qualifications:
+
+    At least 5+ years within backend development, with great experience in python
+    Experience with SQL and NoSQL databases
+    Experience with web scraping
+    Experience with API development
+    Bonus: Experience working with containerization and running computations in cloud environments. We mostly use Azure
+    Bonus: Experience with data processing in Databricks
+    Bonus: Previous work from start-up companies
+
+
+What we offer:
+
+    A highly entrepreneurial environment with flat hierarchies and short decision paths.
+    We are 25 colleagues today, have clients in fourteen markets, and have an ambition to build a fast-growing organization. You’ll be part of the ride from nearly the start and learn how to build a software company from the ground up.
+    Outstanding engineering and product team, shipping high-quality product incredibly fast.
+    Salary and packages are tailored individually based on experience and fit for the team.
+    An office full of talented, mission-driven people.
+    A position with direct feedback from customers.
+    A strong, collaborative work environment with a history of successful startups.
+    Enjoyable lunches at our Cantina and a multicultural team that communicates in English.
+    Office stocked with free snacks, soda, and coffee
+    Located at Fiolstræde, centrally in Copenhagen near Nørreport. (25 min from Malmö)
+    Diverse team with members from (Denmark, Sweden, Chile, Iran, Spain, Ukraine, and India)
+
+
+Application process:
+
+The application process begins with a review by our team. Following this, candidates will undergo interviews conducted via voice or video calls. Successful candidates will then be invited for an in-person interview where they will have the opportunity to meet the team to ensure a good fit. During this process there may be a technical test and TT38 personality test. We care about finding the right person for the job with best fit for the team. The final stage includes contract negotiations. The entire hiring process is expected to take anywhere from a few days to a couple of weeks.
+
+
+For any questions related to this role please contact our CTO, Joakim Erlandsson. +46 705 53 53 95

job-postings/04-01-2024/1.txt

@@ -0,0 +1,49 @@
+
+About the job
+
+Defence projects, unlocked.
+
+
+At Defence.Works, we match you with the industry's top R&D projects and help you deliver impact for a safer tomorrow.
+
+
+With Defence.Works, you will get access to interesting projects without consuming time on sales and project hunting. Our platform allows us to keep our organization structure thin, which means less internal costs, more value to customers, and higher rates for you.
+
+
+As an Artificial Intelligence Engineer you will join our trusted network to work on cutting-edge R&D projects in the defense sector, contributing to innovative solutions that enhance safety and security for a safer tomorrow. You will bring deep expertise in machine learning and artificial intelligence, ideally with prior experience in developing autonomous products, radar systems, or applications that heavily leverage AI capabilities. The ideal candidate will have a proven track record of building secure, high-performing solutions designed to meet the defense industry's rigorous standards.
+
+
+Your presence can be in Northern Europe (Finland, Sweden, Norway, Denmark), United Kingdom, or Germany and you are eligible to pass security-clearance before starting on a project. Please notice that consultants might need to pass security clearance before starting on a project.
+
+
+Responsibilities:
+
+    Design, implement, and optimize AI models and algorithms for defense sector applications, including autonomous systems, radars, and AI-driven decision-making tools.
+    Develop secure, high-performance software solutions tailored to meet the stringent requirements of defense industry projects.
+    Identify and address vulnerabilities in AI systems, ensuring robust cybersecurity measures are integrated throughout the development lifecycle.
+    Collaborate with cross-functional teams to align AI capabilities with overall system architectures, ensuring seamless integration and performance.
+    Conduct rigorous testing, validation, and documentation of AI solutions to ensure reliability, scalability, and compliance with industry standards.
+
+
+Qualifications:
+
+    Master's or Ph.D. in Computer Science, Machine Learning, or related field
+    Strong skills in Pattern Recognition, Neural Networks, and Algorithms.
+    Experience from developing products which rely on AI/ML capabilities such as autonomous systems, radar technologies, or other AI-reliant defense applications.
+    Expertise in machine learning frameworks (e.g., TensorFlow, PyTorch) and programming languages such as Python, C++, or Java.
+    Familiarity with secure system design, cybersecurity principles, and compliance with defense-specific regulations.
+
+
+Additional Requirements:
+
+    Security certifications or training (e.g., CISSP, CEH) are highly desirable.
+
+
+Next steps:
+
+    Press "Apply"
+    Submit your resume on Hashlist platform (takes 5 minutes). Hashlist is our parental company which infrastructure we are utilizing to manage the profiles
+    We will review your profile and if your profile matches our criteria
+    We call you and make sure that your expectations and career ambitions are aligned with our offer.
+    You will get accepted to our trusted network of partners and our clients and receive offers from different clients who are looking for your expertise
+

job-postings/04-01-2024/2.txt

@@ -0,0 +1,49 @@
+
+About the job
+
+Job Description
+
+We are on the journey to transform our digital capabilities, bringing core business processes, people, data & technology together - an enabler for IKEA to become an even better home furnishing retailer in the future. A journey that needs passionate people who embrace change, dare to question and want to make a difference. If that sounds like you, come and join us. Together we can do great things! 
+
+We're on the lookout for passionate doers, idea makers, creative thinkers - people who want to help IKEA find answers to everyday problems in people's lives using Spatial Computing. Do you want to be part of making it happen? Then keep on reading.
+
+The Team 
+
+The Spatial Computing Team drives the digital innovation work in the spatial computing area for all IKEA companies. We are looking for a Research Engineer with knowledge of Spatial Computing / XR Development. In this position you will use your technical expertise to find, explore, evaluate and transfer innovation insights. You are someone with an innovative mind and lateral thinking. Someone who thinks virtual worlds are cool but helping real people is even better. You know why robots and pie go well together and why point clouds on a sunny day don't matter at all.
+
+Your Main Responsibilities Will Be To
+
+    Explore and evaluate new technology and its possibility to improve IKEA customer and co-worker experience 
+    Develop real-time 3D applications to serve as inspiring proof of concepts 
+    Collaborate with external experts and innovators in explorations of technical solutions 
+    Collect and share expertise with IKEA stakeholders 
+
+About You
+
+The person we're looking for is someone passionate about the future of 3D graphics. A person who wants to use game engine for more than just for games. Someone who likes the idea of conceptualizing the utopian future of the digital - human interfaces, to interact with people in a more natural way than ever before. Someone who thinks Swedish meatballs have the potential to taste even better in mixed reality. We're looking for a person who can break down high-level concepts like these and explore them singlehandedly or as part of a team.
+
+To be successful in this role, the following knowledge, skills and experiences would be valuable: 
+
+    Background in real-time 3D graphics with deep interest in the future of Real Time Graphics (VR/MR/AR). 
+    Understanding of graphics pipelines related to real-time 3D environments. 
+    Passion for how sound and audio design and haptics can be used to elevate immersive experiences 
+    Experience with Unreal and/or Unity and/or native IOS/Android 3D development and/or Web based 3D engines 
+    Experience with mobile application development and deployment. 
+    Programming skills building applications communicating with back-ends and building applications of interactive worlds using game engines and 3D Graphics. 
+    Good knowledge of at least 2 different core programming languages such as C,C#, Python, C++ or JavaScript 
+    You'll have a passion for sharing the knowledge you've acquired, with the ability to communicate with both technical and non-technical stakeholders. 
+    Ability to formulate new ideas surrounding technological innovations. 
+    Ability to discuss problems with program code examples. 
+    Strong collaboration skills, with experience developing solutions alongside other teammates. 
+
+Additional Information
+
+This role is full-time (40 hours per week) and based in Älmhult. This role sits in the Range Operations and reports to Innovation Manager. If you would like to know more about the role, please connect with [email protected] or if you have questions about the recruitment process, please connect with [email protected]
+
+At IKEA, we are looking for people who believe everyone deserves a seat at the table. You’re welcome no matter where you come from, what you believe, and what you look like. We don’t even care how you have furnished your home. We’re interested in you simply because you’re you. Even if your experience doesn’t align perfectly with every qualification in the job description, we encourage you to apply anyway. We believe that people’s different perspectives, backgrounds, and personalities make us better at understanding our customers dreams and needs. At IKEA, we’re all on the same project.
+
+If you have a special need that requires accommodation in the recruitment process, just let us know.
+
+Please note that due to the upcoming holiday season in December, our recruitment process may take a bit longer than usual. We will do our best to keep all candidates updated on their status before any breaks. Thank you for your patience during this time! 
+
+Interested? Submit your CV and let us know why you would be a good fit for this role, in English, by 7th of January 2025.

job-postings/04-01-2024/3.txt

@@ -0,0 +1,61 @@
+
+About the job
+
+Rillion is seeking a skilled AI Engineer to join our innovative Data/AI team. In this role, you’ll be pivotal in developing machine learning models that drive AI-powered AP (Accounts Payable) automation and enhance our products. Our team consists of data engineers, an AI/ML lead, a data lead, a UX designer, a product manager, and software engineers. Together, we collaborate across the entire product lifecycle—from brainstorming and design to implementation—unlocking AI’s potential in AP automation. If you’re an experienced AI/ML developer who thrives in a dynamic environment, we’d love to have you on board!
+
+
+Responsibilities:
+
+The AI Engineer plays a key role in developing and implementing AI technologies into our SaaS products, with a focus on technical execution rather than leadership. This role is critical in building, integrating, and optimizing AI/ML solutions to meet product goals.
+
+
+    Develop and implement scalable AI/ML models that support product objectives.
+    Collaborate closely with product, engineering, and data teams to integrate AI features into our products.
+    Stay up-to-date with emerging AI technologies and contribute to experimentation and innovation.
+    Build and maintain effective AI/ML pipelines and deployment infrastructure.
+
+
+What You'll Bring:
+
+    2+ years hands-on experience with putting self-developed machine learning solutions into a production environment.
+    High proficiency in Python and AI/ML frameworks, i.e. Pytorch.
+    Familiarity with working with containers in a cloud environment like Azure, AWS, and/or GCP, with experience scaling AI solutions in production.
+    A curious mindset with strong collaboration skills who is comfortable in environments without clear-cut processes.
+    Master's degree in engineering or similar.
+
+
+Bonus skills:
+
+    Cloud Ops and IaC tools such as Terraform.
+    MLOps best practices and tools like Databricks.
+    Knowledge of working with visually rich documents (VRDs) and generative AI.
+    Experience working with RAG, LLM evaluation, API-driven microservices, cache management and production-level software.
+
+
+What we offer:
+
+    Opportunity to work in a dynamic growth company
+    Talented colleagues ready to support the success in your career path
+    Social events with your colleagues (breakfast, candy-time, afterwork etc.)
+    A collection of different benefits, including a generous pension and insurance package
+    Hybrid working model, 2 days per week in the office
+    Come and enjoy our beautiful office in central Stockholm (on the 14th floor, with amazing views)
+
+
+The recruitment process:
+
+We review applications and invite for interviews continuously. A background check will be conducted on final candidates, pre-employment. If you have any questions regarding the role, please contact recruiter Victoria Stjernström at [email protected]
+
+
+About Rillion:
+
+We are a global company founded in Sweden with 30 years’ experience in the AP Automation industry. We help finance professionals transform how they manage invoices by digitalizing and automating the entire process.
+
+
+By removing the manual steps of invoice handling, we enable finance teams to save time and effort, reducing the possibility of human error. Because we’re AP professionals ourselves, we understand how to give our customers everything they need, and nothing they don’t.
+
+
+Together with our owners at Altor, we´re on a journey to expand in our home markets, while entering new territories. To complete our mission, we need more talented people!
+
+
+Rillion is an equal opportunity employer. We believe that diversity is integral to our success, and do not discriminate based on race, color, religion, age, or any other basis protected by law.

requirements.txt

@@ -5,4 +5,5 @@ httpx
 idna
 langchain_openai
 python-dotenv
-torch @ https://download.pytorch.org/whl/cpu
+torch
+spacy

tag-posting.py

@@ -0,0 +1,40 @@
+import spacy
+import re
+
+nlp = spacy.load("en_core_web_sm")
+
+def split_text_recursively(text):
+    if '\n' not in text:
+        return [text]
+    parts = text.split('\n', 1)
+    return [parts[0]] + split_text_recursively(parts[1])
+
+def parse_post(path):
+
+    # Read the file
+
+    with open(path, 'r') as file:
+        text = file.read()
+
+    # Sentence tokenization
+
+    str_list = split_text_recursively(text)
+    str_list = [i.strip() for i in str_list]
+    str_list = list(filter(None, str_list))
+
+    count = 0
+    sents = []
+
+    for line in str_list:
+        doc = nlp(line)
+        for sent in doc.sents:
+            print(f"{sent.text}")
+            sents.append(sent.text)
+
+    # Skill/knowledge extraction
+    
+    
+
+
+path = './job-postings/03-01-2024/2.txt'
+parse_post(path)

tagging.py

@@ -13,8 +13,44 @@ from transformers import AutoTokenizer, AutoModelForTokenClassification
 import torch
 import sys
 from tabulate import tabulate
+import spacy
+import re
+
 load_dotenv(".env")
 
+nlp = spacy.load("en_core_web_sm")
+
+def split_text_recursively(text):
+    if '\n' not in text:
+        return [text]
+    parts = text.split('\n', 1)
+    return [parts[0]] + split_text_recursively(parts[1])
+
+
+def tokenize_to_sent(path):
+
+    # Read the file
+
+    with open(path, 'r') as file:
+        text = file.read()
+
+    # Sentence tokenization
+
+    str_list = split_text_recursively(text)
+    str_list = [i.strip() for i in str_list]
+    str_list = list(filter(None, str_list))
+
+    count = 0
+    sents = []
+
+    for line in str_list:
+        doc = nlp(line)
+        for sent in doc.sents:
+            # print(f"{sent.text}")
+            sents.append(sent.text)
+    
+    return sents
+
 
 ### LLM-based tag extraction with few-shot learning
 
@@ -88,10 +124,63 @@ def convert(text):
 
     skill_cls = [mapping[i.item()] for i in skill_cls]
     knowledge_cls = [mapping[i.item()] for i in knowledge_cls]
-    return skill_cls, knowledge_cls
+
+    if len(decoded_tokens) != len(skill_cls) or len(decoded_tokens) != len(knowledge_cls):
+        raise ValueError("Error: Length mismatch")
+
+    return skill_cls, knowledge_cls, decoded_tokens
+
+
+from transformers import pipeline
+pipe = pipeline("token-classification", model="jjzha/jobbert_knowledge_extraction")
+
+def convert2(text):
+    output = pipe(text)
+    tokens = [i['word'] for i in output]
+    skill_cls = [i['entity'] for i in output]
+    knowledge_cls = [i['entity'] for i in output]
+
+    return skill_cls, knowledge_cls, tokens
+    
+
+
+
+def tag_posting(path, llm_extract = True):
+
+    # Reading & sentence tokenization
+    sents = tokenize_to_sent(path)
+
+    for sent in sents:
+        # print(f"Sent: {sent}")
+        skill_cls, knowledge_cls, tokens = convert(sent)
+           
+
+    # Pre-trained
+    # skill_cls, knowledge_cls, _ = convert(text)
+
+    if llm_extract:
+
+        # LLM-based tag extraction
+        tokens, output = extract_tags(text, tokenize=True)
+        table = zip(tokens, output['skill_labels'], output['knowledge_labels'], skill_cls, knowledge_cls)
+        headers = ["Token", "Skill Label", "Knowledge Label", "Pred Skill Label", "Pred Knowledge Label"]
+        print(tabulate(table, headers=headers, tablefmt="pretty"))
+
+    else:
+
+        # Only pre-trained
+        table = zip(tokens, output['skill_labels'], output['knowledge_labels'])
+        headers = ["Token", "Skill Label", "Knowledge Label"]
+        print(tabulate(table, headers=headers, tablefmt="pretty"))
+
 
 
 if __name__ == "__main__":
+
+    path = './job-postings/03-01-2024/1.txt'
+    tag_posting(path, llm_extract = False)
+
+    quit()
     text = input('Enter text: ')
 
     # LLM-based tag extraction