finalizing

.github/workflows/scraping.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install -r all-requirements.txt
 
     - name: Run LinkedIn Scraping Script
       env:

.github/workflows/trends.yml

@@ -0,0 +1,44 @@
+name: Run trend graph Script
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run every 30 minutes
+    #- cron: '*/30 * * * *'
+    - cron: '0 0 * * 0'
+
+jobs:
+  run-scraper:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run trend graph Script
+      env:
+        RAPID_API_KEY: ${{ secrets.RAPID_API_KEY }}
+      run: |
+        python trend_graph.py
+    - name: List plots folder
+      run: ls -R plots || echo "plots folder not found"
+    - name: Commit and Push Changes
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git add job-postings
+        git commit -m "Add plots generated by scrip"
+        git push
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/visualization.yml

@@ -0,0 +1,42 @@
+name: Run Visualization Script
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run every 30 minutes
+    #- cron: '*/30 * * * *'
+    - cron: '0 0 * * 0'
+
+jobs:
+  run-scraper:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run Visualization Script
+      run: |
+        python embedding_gen.py
+    - name: List plots folder
+      run: ls -R plots || echo "plots not found"
+    - name: Commit and Push Changes
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git add plots
+        git commit -m "Add plots generated by script"
+        git push
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

README.md

@@ -10,7 +10,11 @@ pinned: false
 ---
 
 
-# Compilation of in-demand tech skills
+# In-demand Skill Monitoring for Machine Learning Industry
+
+## About
+
+This projects strives to monitor in-demand skills for machine learning roles based in Stockholm, Sweden. 
 
 # Project outline
 

embedding_gen.py

@@ -87,7 +87,13 @@ def visualize3D(reduced_embeddings, labels, skills, n_clusters, output_folder, d
     # fig.show()
     return fig
 
-if __name__ == "__main__":
+# Main execution
+base_folder = "./tags"
+output_folder = "./plots"
+specific_date = "03-01-2024"  # Example date folder to process
+# Get today's date in the desired format
+# specific_date = datetime.now().strftime("%d-%m-%Y")
+n_clusters = 5
 
     # Main execution
     base_folder = "./tags"

requirements.txt

@@ -7,4 +7,8 @@ langchain_openai
 python-dotenv
 torch
 spacy
-plotly
+sentence-transformers
+umap-learn
+scikit-learn
+matplotlib
+plotly

trend_graph.py

@@ -46,6 +46,7 @@ top_skills = total_counts.nlargest(3).index
 
 # Step 5: Plot and save separate graphs for the top 3 skills
 for skill in top_skills:
+    print(f"Trend of {skill} Over Time")
     plt.figure(figsize=(8, 5))
     plt.plot(df.index, df[skill], marker="o", label=skill)