section extraction to mongodb is working through the notebook but needs to be moved to a function.

notebooks/mongo_section_extraction.ipynb

@@ -0,0 +1,134 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from multiprocessing import process\n",
+    "import pandas as pd\n",
+    "import datetime as dt\n",
+    "import http.client\n",
+    "import json\n",
+    "import urllib.parse\n",
+    "import os\n",
+    "from pymongo import MongoClient\n",
+    "import sys\n",
+    "sys.path.append('../')\n",
+    "\n",
+    "from utils import parse_description\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mongodb_conn = os.getenv('MONGODB_CONNECTION_STRING')\n",
+    "client = MongoClient(mongodb_conn)\n",
+    "db = client.job_search_db\n",
+    "collection = db['sf_bay_test_jobs']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "company_overview=CompanyOverview(about='Global Cloud Services is a cloud computing company with a location in Palo Alto, CA', mission_and_values='The company values data-driven decision making and uses machine learning to drive business value', size='Not specified', locations='Palo Alto, CA') role_summary=RoleSummary(title='Data Scientist', team_or_department='Multi-Cloud team', role_type='Full-time', remote='Not specified') responsibilities_and_qualifications=ResponsibilitiesAndQualifications(responsibilities=['Explore data sources, implement data exploration, pre-processing and data cleansing on historical data', 'Develop statistical/machine learning models based on pre-processed data as a proof-of-concept', 'Assess and optimize model quality based on technical level by tuning hyper-parameters, settings and/or changing models', 'Integrate statistical/machine learning models into products & services', 'Work closely with product managers, developers, and engineers to ensure that models will be used in products or services', 'Evaluate, improve/refine models by tuning parameters, adjusting data sources, or model approaches, to provide business value'], required_qualifications=[\"Bachelor's degree in Computer Science, Engineering, Mathematics or a related field of study and 4 years of experience\", \"Master's degree and 2 years of experience\"], preferred_qualifications=['Not specified']) compensation_and_benefits=CompensationAndBenefits(salary_or_pay_range='$80K -- $100K', bonus_and_equity='Not specified', benefits_and_perks=['Not specified'])\n",
+      "company_overview=CompanyOverview(about='Stanford Health Care', mission_and_values='delivering value and an exceptional experience for our patients and families', size='N/A', locations='1830 Embarcadero Road, PALO ALTO, California') role_summary=RoleSummary(title='Senior Biomedical Informatics Data Scientist', team_or_department='IT RESEARCH Technology & Digital Solutions', role_type='Full-time', remote='Hybrid') responsibilities_and_qualifications=ResponsibilitiesAndQualifications(responsibilities=['Work closely with the data science and engineering team on data quality analysis', 'Develop processes to measure and ensure quality, completeness, integrity and compliance of institutional research data assets', 'Identify best practices in the technical community and help to shape and implement policies that enhance data quality, compliance and customer support', 'Develop ETL (extract, transform, load) specifications to go from raw data to research ready datasets', 'Employ new and existing tools to interpret, analyze, and visualize relationships in data', 'Analyze and incorporate external data sets that may augment the power of clinical data', 'Represent Stanford through presentations at technical conferences, consortiums, participation in standard committees, working groups and other venues', 'Engage in other departmental activities to ensure an inclusive and transparent work culture'], required_qualifications=[\"Bachelor's degree in a scientific field\", 'Strong analytical skills', 'Experience with data manipulation and integration, databases, and statistics', 'Fluency with data science programming paradigms such as Jupyter notebooks, SQL, Python or R'], preferred_qualifications=[\"Master's or PhD\", '4+ years of related experience']) compensation_and_benefits=CompensationAndBenefits(salary_or_pay_range='$55.80 - $73.92 per hour', bonus_and_equity='N/A', benefits_and_perks=['N/A'])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# for every document in a given collection get the job_id, title, company_name, location, description, detected_extensions, retrieve_date\n",
+    "# then transform the description into the JobDescription object and save each lower object as a attribute of the Job object\n",
+    "# how can i find only 5 documents\n",
+    "\n",
+    "# get the first 5 documents\n",
+    "\n",
+    "# loop through the collection\n",
+    "for document in collection.find().limit(2):\n",
+    "    job_id = document['job_id']\n",
+    "    title = document['title']\n",
+    "    company_name = document['company_name']\n",
+    "    location = document['location']\n",
+    "    description = document['description']\n",
+    "    detected_extensions = document['detected_extensions']\n",
+    "    retrieve_date = document['retrieve_date']\n",
+    "    job_object = parse_description.extract_desc_fields(description)\n",
+    "    \n",
+    "    #insert all of the above as a new document in \"sf_bay_test_jobs_cleaned\" collection\n",
+    "    db['sf_bay_test_jobs_cleaned'].insert_one({\n",
+    "        'job_id': job_id,\n",
+    "        'title': title,\n",
+    "        'company_name': company_name,\n",
+    "        'location': location,\n",
+    "        'description': description,\n",
+    "        'detected_extensions': detected_extensions,\n",
+    "        'retrieve_date': retrieve_date,\n",
+    "        'company_overview': job_object.company_overview.dict(),\n",
+    "        'role_summary': job_object.role_summary.dict(),\n",
+    "        'responsibilities_and_qualifications': job_object.responsibilities_and_qualifications.dict(),\n",
+    "        'compensation_and_benefits': job_object.compensation_and_benefits.dict()\n",
+    "    })\n",
+    "       \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DeleteResult({'n': 2, 'electionId': ObjectId('7fffffff00000000000000e2'), 'opTime': {'ts': Timestamp(1715378977, 13), 't': 226}, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1715378977, 14), 'signature': {'hash': b'F\\x92\\xb5\\xce\\xc1\\x189\\x9b\\x00\\xc6\\xe43\\xaa\\xc0\\x9a\\xfe\\xd5\\xdf\\x80\\x14', 'keyId': 7314031900077588486}}, 'operationTime': Timestamp(1715378977, 13)}, acknowledged=True)"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "db['sf_bay_test_jobs_cleaned'].delete_many({})\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "datajobs",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}