Merge pull request #12 from brandonscholten/merge-upstream-changes

Merge upstream changes

Author: cnewman

.gitignore

@@ -2,4 +2,4 @@ output/
 __pycache__/
 code2vec/
 cache/
-input.txt
+input.txt

Dockerfile

@@ -1,11 +1,19 @@
 FROM python:3.12-slim
 
+#argument to enable GPU accelaration
+ARG GPU=false
+
 # Install (and build) requirements
 COPY requirements.txt /requirements.txt
-RUN apt-get update && \
-    apt-get install -y git curl && \
+COPY requirements_gpu.txt /requirements_gpu.txt
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    apt-get update --fix-missing && \
+    apt-get install --allow-unauthenticated -y git curl && \
     pip install -r requirements.txt && \
-    rm -rf /var/lib/apt/lists/*
+    if [ "$GPU" = true ]; then \
+        pip install -r requirements_gpu.txt; \
+    fi && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY . .
 RUN pip install -e .
@@ -71,4 +79,4 @@ CMD date; \
     echo "Running..."; \
     /main -r --words words/abbreviationList.csv
 
-ENV TZ=US/Michigan
+ENV TZ=US/Michigan

README.md

@@ -63,9 +63,13 @@ options:
 
 `./main -r` will start the server, which will listen for identifier names sent via HTTP over the route:
 
-http://127.0.0.1:8080/{cache_selection}/{identifier_name}/{code_context}
+http://127.0.0.1:8080/{identifier_name}/{code_context}/{database_name (optional)}
 
-"cache selection" will save results to a separate cache if it is set to "student"
+"database name" specifies an sqlite database to be used for result caching and data collection. If the database specified does not exist, one will be created. 
+
+You can check wehther or not a database exists by using the `/probe` route by sending an HTTP request like this:
+
+http://127.0.0.1:5000/probe/{database_name}
 
 "code context" is one of:
 - FUNCTION
@@ -76,11 +80,11 @@ http://127.0.0.1:8080/{cache_selection}/{identifier_name}/{code_context}
 
 For example:
 
-Tag a declaration: ``http://127.0.0.1:8080/cache/numberArray/DECLARATION``
+Tag a declaration: ``http://127.0.0.1:8000/numberArray/DECLARATION/database``
 
-Tag a function: ``http://127.0.0.1:8080/cache/GetNumberArray/FUNCTION``
+Tag a function: ``http://127.0.0.1:8000/GetNumberArray/FUNCTION/database``
 
-Tag an class: ``http://127.0.0.1:8080/cache/PersonRecord/CLASS``
+Tag an class: ``http://127.0.0.1:8000/PersonRecord/CLASS/database``
 
 #### Note
 Kebab case is not currently supported due to the limitations of Spiral. Attempting to send the tagger identifiers which are in kebab case will result in the entry of a single noun. 
@@ -156,4 +160,3 @@ Find our other research [at our webpage](https://www.scanl.org/) and check out t
 This project uses WordNet to perform a dictionary lookup on the individual words in each identifier:
 
 Princeton University "About WordNet." [WordNet](https://wordnet.princeton.edu/). Princeton University. 2010
-

compose.yml

@@ -20,3 +20,4 @@ services:
       - words:/words
     ports:
       - "${PORT-8080}:5000"
+    restart: always

requirements.txt

@@ -17,7 +17,7 @@ filelock==3.17.0
 flair==0.15.0
 Flask==3.1.0
 fonttools==4.55.6
-fsspec==2024.12.0
+fsspec==2023.5.0
 ftfy==6.3.1
 gdown==5.2.0
 gensim==4.3.3
@@ -42,18 +42,6 @@ mpmath==1.3.0
 networkx==3.4.2
 nltk==3.9.1
 numpy==1.26.4
-nvidia-cublas-cu12==12.4.5.8
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cufft-cu12==11.2.1.3
-nvidia-curand-cu12==10.3.5.147
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-nccl-cu12==2.21.5
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nvtx-cu12==12.4.127
 packaging==24.2
 pandas==2.2.3
 pillow==11.1.0
@@ -93,7 +81,6 @@ torch==2.5.1
 tqdm==4.67.1
 transformer-smaller-training-vocab==0.4.0
 transformers==4.48.1
-triton==3.1.0
 typing_extensions==4.12.2
 tzdata==2025.1
 urllib3==2.3.0

requirements_gpu.txt

@@ -0,0 +1,12 @@
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.1.1.17
+nvidia-cufft-cu12==11.2.1.3
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-nccl-cu12==2.23.4
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.4.127

serve.json

@@ -3,4 +3,4 @@
     "port": 8080,
     "protocol": "https",
     "words":""
-}
+}

src/classifier_multiclass.py

@@ -121,10 +121,12 @@ def perform_classification(X, y, results_text_file, output_directory, TrainingAl
     Returns:
         None
     """
+
     X_train, X_test, y_train, y_test, X_train_original, X_test_original = build_datasets(X, y, output_directory, trainingSeed)
     labels = np.unique(y_train, return_counts=False)
 
     algoData = TrainTestvalidationData(X_train, X_test, y_train, y_test, X_train_original, X_test_original, labels)
+    
     results_text_file.write("Training Seed: %s\n" % trainingSeed)
     results_text_file.write("Classifier Seed: %s\n" % classifierSeed)
 
@@ -151,7 +153,6 @@ def write_importances(results_text_file, feature_names, presult, metric_name):
         results_text_file.write(f"{feature},{value}\n")
     results_text_file.write("\n")
 
-
 def analyzeGradientBoost(results_text_file, output_directory, scorersKey, algoData, classifierSeed, trainingSeed, columns_to_drop):
     """
     Analyze a GradientBoostingClassifier for classification and report results.
@@ -180,6 +181,7 @@ def analyzeGradientBoost(results_text_file, output_directory, scorersKey, algoDa
         print("GradientBoostingClassifier")
 
         # Drop SPLIT_IDENTIFIER and WORD columns from X_train
+
         X_train_dropped = algoData.X_train.drop(columns=columns_to_drop, errors='ignore')
 
         max_threads = max(1, multiprocessing.cpu_count() - 3)

src/download_code2vec_vectors.py

@@ -14,6 +14,7 @@ def create_directory_if_not_exists(directory_path):
     if not os.path.exists(directory_path):
         os.makedirs(directory_path)
 
+
 def download_files():
 
     nltk.download('averaged_perceptron_tagger_eng')

src/feature_generator.py

@@ -274,6 +274,7 @@ def createFeatures(data: pd.DataFrame, feature_list: List[str], modelTokens = No
     'POS': 'NOUN',    # Possessive ending
     'RP': 'NOUN',     # Particle
     'X': 'NOUN',      # Unknown
+
     'START':'START',
     'END':'END',
 
@@ -456,6 +457,7 @@ def compute_similarity(verb_vector, target_word, model):
     similarity = np.dot(verb_vector, target_word_vector)
     return similarity
 
+
 def contrastive_embedding(target_vector, contrast_vectors, beta=0.1):
     """
     Adjust a target embedding by pushing it away from contrast embeddings.
@@ -557,6 +559,7 @@ def createDeterminerVectorFeature(data, model):
         pandas.DataFrame: The input DataFrame with an additional 'DET_SCORE' column.
     """
     words = data["WORD"]
+    
     # Convert sets to lists before merging
     non_determiners = list(nouns) + list(verbs) + list(prepositions) +  list(conjunctions)
 
@@ -570,6 +573,7 @@ def createDeterminerVectorFeature(data, model):
 
     # Compute similarity
     scores = pd.DataFrame([compute_similarity(adjusted_determiner_vector, word.lower(), model) for word in words])
+
     scores.columns = ['DET_SCORE']
 
     data = pd.concat([data, scores], axis=1)
@@ -592,6 +596,7 @@ def createPrepositionVectorFeature(data, model):
         pandas.DataFrame: The input DataFrame with an additional 'PREP_SCORE' column.
     """
     words = data["WORD"]
+
     # Convert sets to lists before merging
     non_prepositions = list(nouns) + list(verbs) + list(determiners) + list(conjunctions)
 
@@ -662,6 +667,7 @@ def createPreambleVectorFeature(name, data, model):
         The actual name of the new column will be 'name'+'PRE_SCORE' (e.g., 'CODEPRE_SCORE', 'METHODPRE_SCORE').
     """
     words = data["WORD"]
+
     # Convert sets to lists before merging
     non_preambles = list(nouns) + list(verbs) + list(determiners) + list(prepositions) + list(determiners)
 
@@ -866,10 +872,12 @@ def createIdentifierClosedSetFeature(data, conjunctions=conjunctions, determiner
         pandas.DataFrame: Updated DataFrame with a 'CONTAINSCLOSEDSET' column.
     """
     closed_set = set(conjunctions) | set(determiners) | set(prepositions)
+
     words = data["WORD"]
     isClosedSet = pd.DataFrame([1 if word in closed_set else 0 for word in words])
     isClosedSet.columns = ["CONTAINSCLOSEDSET"]
     data = pd.concat([data, isClosedSet], axis=1)
+    
     return data
 
 def createIdentifierContainsVerbFeature(data, verbs=verbs):
@@ -884,10 +892,12 @@ def createIdentifierContainsVerbFeature(data, verbs=verbs):
         pandas.DataFrame: Updated DataFrame with a 'CONTAINSVERB' column.
     """
     verb_set = set(verbs)
+    
     words = data["WORD"]
     isVerb = pd.DataFrame([1 if word in verb_set else 0 for word in words])
     isVerb.columns = ["CONTAINSVERB"]
     data = pd.concat([data, isVerb], axis=1)
+    
     return data
 
 def addMorphologicalPluralFeature(data):