Merge pull request #179 from henk717/united

1.19.2

Author: henk717

aiserver.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 #==================================================================#
 # KoboldAI
-# Version: 1.19.1
+# Version: 1.19.2
 # By: The KoboldAI Community
 #==================================================================#
 
@@ -125,6 +125,7 @@ class colors:
         ["NSFW Models", "nsfwlist", "", True],
         ["Untuned OPT", "optlist", "", True],
         ["Untuned GPT-Neo/J", "gptneolist", "", True],
+        ["Untuned Pythia", "pythialist", "", True],
         ["Untuned Fairseq Dense", "fsdlist", "", True],
         ["Untuned Bloom", "bloomlist", "", True],
         ["Untuned XGLM", "xglmlist", "", True],
@@ -154,6 +155,7 @@ class colors:
         ["OPT Nerys 6B V2 (Hybrid)", "KoboldAI/OPT-6B-nerys-v2", "16GB", False],
         ["Janeway FSD 6.7B", "KoboldAI/fairseq-dense-6.7B-Janeway", "16GB", False],
         ["Janeway Neo 6B", "KoboldAI/GPT-J-6B-Janeway", "16GB", False],
+        ["Qilin Lit 6B (SFW)", "rexwang8/qilin-lit-6b", "16GB", False],       
         ["Janeway Neo 2.7B", "KoboldAI/GPT-Neo-2.7B-Janeway", "8GB", False],
         ["Janeway FSD 2.7B", "KoboldAI/fairseq-dense-2.7B-Janeway", "8GB", False],
         ["Nerys FSD 2.7B (Hybrid)", "KoboldAI/fairseq-dense-2.7B-Nerys", "8GB", False],
@@ -183,12 +185,31 @@ class colors:
         ],
     'gptneolist': [
         ["GPT-NeoX 20B", "EleutherAI/gpt-neox-20b", "64GB", False],
+        ["Pythia 13B (NeoX, Same dataset)", "EleutherAI/pythia-13b", "32GB", False],
         ["GPT-J 6B", "EleutherAI/gpt-j-6B", "16GB", False],
         ["GPT-Neo 2.7B", "EleutherAI/gpt-neo-2.7B", "8GB", False],
         ["GPT-Neo 1.3B", "EleutherAI/gpt-neo-1.3B", "6GB", False],
+        ["Pythia 800M (NeoX, Same dataset)", "EleutherAI/pythia-800m", "4GB", False],
+        ["Pythia 350M (NeoX, Same dataset)", "EleutherAI/pythia-350m", "2GB", False],
         ["GPT-Neo 125M", "EleutherAI/gpt-neo-125M", "2GB", False],
         ["Return to Main Menu", "mainmenu", "", True],
         ],
+    'pythialist': [
+        ["Pythia 13B Deduped", "EleutherAI/pythia-13b-deduped", "32GB", False],
+        ["Pythia 13B", "EleutherAI/pythia-13b", "32GB", False],
+        ["Pythia 6.7B Deduped", "EleutherAI/pythia-6.7b-deduped", "16GB", False],
+        ["Pythia 6.7B", "EleutherAI/pythia-6.7b", "16GB", False],
+        ["Pythia 1.3B Deduped", "EleutherAI/pythia-1.3b-deduped", "6GB", False],
+        ["Pythia 1.3B", "EleutherAI/pythia-1.3b", "6GB", False],
+        ["Pythia 800M", "EleutherAI/pythia-800m", "4GB", False],
+        ["Pythia 350M Deduped", "EleutherAI/pythia-350m-deduped", "2GB", False],
+        ["Pythia 350M", "EleutherAI/pythia-350m", "2GB", False],        
+        ["Pythia 125M Deduped", "EleutherAI/pythia-125m-deduped", "2GB", False],
+        ["Pythia 125M", "EleutherAI/pythia-125m", "2GB", False],
+        ["Pythia 19M Deduped", "EleutherAI/pythia-19m-deduped", "1GB", False],
+        ["Pythia 19M", "EleutherAI/pythia-19m", "1GB", False],
+        ["Return to Main Menu", "mainmenu", "", True],
+        ],
     'gpt2list': [
         ["GPT-2 XL", "gpt2-xl", "6GB", False],
         ["GPT-2 Large", "gpt2-large", "4GB", False],
@@ -452,6 +473,7 @@ def emit(*args, **kwargs):
         return _emit(*args, **kwargs)
     except AttributeError:
         return socketio.emit(*args, **kwargs)
+utils.emit = emit
 
 # marshmallow/apispec setup
 from apispec import APISpec
@@ -756,6 +778,12 @@ def getmodelname():
         modelname = vars.model
         return modelname
 
+#==================================================================#
+# Get hidden size from model
+#==================================================================#
+def get_hidden_size_from_model(model):
+    return model.get_input_embeddings().embedding_dim
+
 #==================================================================#
 # Breakmodel configuration functions
 #==================================================================#
@@ -873,7 +901,7 @@ def device_config(config):
                     print(f"{colors.RED}Please enter an integer between -1 and {n_layers}.{colors.END}")
 
     logger.init_ok("Final device configuration:", status="Info")
-    device_list(n_layers)
+    device_list(n_layers, primary=breakmodel.primary_device)
 
     # If all layers are on the same device, use the old GPU generation mode
     while(len(breakmodel.gpu_blocks) and breakmodel.gpu_blocks[-1] == 0):
@@ -989,7 +1017,7 @@ def loadmodelsettings():
     if("nobreakmodel" in js):
         vars.nobreakmodel = js["nobreakmodel"]
     if("sampler_order" in js):
-        sampler_order = vars.sampler_order
+        sampler_order = js["sampler_order"]
         if(len(sampler_order) < 7):
             sampler_order = [6] + sampler_order
         vars.sampler_order = sampler_order
@@ -1127,7 +1155,7 @@ def processsettings(js):
     if("andepth" in js):
         vars.andepth = js["andepth"]
     if("sampler_order" in js):
-        sampler_order = vars.sampler_order
+        sampler_order = js["sampler_order"]
         if(len(sampler_order) < 7):
             sampler_order = [6] + sampler_order
         vars.sampler_order = sampler_order
@@ -1354,6 +1382,8 @@ def general_startup(override_args=None):
         args = parser.parse_args(shlex.split(os.environ["KOBOLDAI_ARGS"]))
     else:
         args = parser.parse_args()
+    
+    utils.args = args
 
     set_logger_verbosity(args.verbosity)
     quiesce_logger(args.quiesce)
@@ -1790,7 +1820,9 @@ def new_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwarg
         if not args.no_aria2:
             utils.aria2_hook(pretrained_model_name_or_path, **kwargs)
         return old_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
-    PreTrainedModel.from_pretrained = new_from_pretrained
+    if(not hasattr(PreTrainedModel, "_kai_patched")):
+        PreTrainedModel.from_pretrained = new_from_pretrained
+        PreTrainedModel._kai_patched = True
     if(hasattr(modeling_utils, "get_checkpoint_shard_files")):
         old_get_checkpoint_shard_files = modeling_utils.get_checkpoint_shard_files
         def new_get_checkpoint_shard_files(pretrained_model_name_or_path, index_filename, *args, **kwargs):
@@ -2424,9 +2456,6 @@ def get_original_key(key):
                 return lazy_load_callback
 
 
-            def get_hidden_size_from_model(model):
-                return model.get_input_embeddings().embedding_dim
-            
             def maybe_low_cpu_mem_usage() -> Dict[str, Any]:
                 if(packaging.version.parse(transformers_version) < packaging.version.parse("4.11.0")):
                     logger.warning(f"Please upgrade to transformers 4.11.0 for lower RAM usage. You have transformers {transformers_version}.")
@@ -2668,7 +2697,9 @@ def new_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwarg
             if not args.no_aria2:
                 utils.aria2_hook(pretrained_model_name_or_path, **kwargs)
             return old_from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
-        PreTrainedModel.from_pretrained = new_from_pretrained
+        if(not hasattr(PreTrainedModel, "_kai_patched")):
+            PreTrainedModel.from_pretrained = new_from_pretrained
+            PreTrainedModel._kai_patched = True
         if(hasattr(modeling_utils, "get_checkpoint_shard_files")):
             old_get_checkpoint_shard_files = modeling_utils.get_checkpoint_shard_files
             def new_get_checkpoint_shard_files(pretrained_model_name_or_path, index_filename, *args, **kwargs):
@@ -2914,7 +2945,7 @@ def lua_startup():
     except lupa.LuaError as e:
         print(colors.RED + "ERROR!" + colors.END)
         vars.lua_koboldbridge.obliterate_multiverse()
-        logger.debug('LUA ERROR: ' + str(e).replace("\033", ""))
+        logger.error('LUA ERROR: ' + str(e).replace("\033", ""))
         logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.")
         exit(1)
     logger.init_ok("LUA bridge", status="OK")
@@ -3055,6 +3086,8 @@ def lua_compute_context(submission, entries, folders, kwargs):
         force_use_txt=True,
         scan_story=kwargs["scan_story"] if kwargs["scan_story"] != None else True,
     )
+    if kwargs["include_anote"] is not None and not kwargs["include_anote"]:
+        anotetxt = ""
     txt, _, _ = calcsubmitbudget(
         len(actions),
         winfo,
@@ -3470,7 +3503,7 @@ def execute_inmod():
         vars.lua_running = False
         emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True)
         sendUSStatItems()
-        logger.debug('LUA ERROR: ' + str(e).replace("\033", ""))
+        logger.error('LUA ERROR: ' + str(e).replace("\033", ""))
         logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.")
         set_aibusy(0)
 
@@ -3487,7 +3520,7 @@ def execute_outmod():
         vars.lua_running = False
         emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True)
         sendUSStatItems()
-        logger.debug('LUA ERROR: ' + str(e).replace("\033", ""))
+        logger.error('LUA ERROR: ' + str(e).replace("\033", ""))
         logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.")
         set_aibusy(0)
     if(vars.lua_koboldbridge.resend_settings_required):
@@ -4907,7 +4940,7 @@ def generate(txt, minimum, maximum, found_entries=None):
             vars.lua_running = False
             emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True)
             sendUSStatItems()
-            logger.debug('LUA ERROR: ' + str(e).replace("\033", ""))
+            logger.error('LUA ERROR: ' + str(e).replace("\033", ""))
             logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.")
         else:
             emit('from_server', {'cmd': 'errmsg', 'data': 'Error occurred during generator call; please check console.'}, broadcast=True)
@@ -5415,7 +5448,7 @@ def tpumtjgenerate(txt, minimum, maximum, found_entries=None):
             vars.lua_running = False
             emit('from_server', {'cmd': 'errmsg', 'data': 'Lua script error; please check console.'}, broadcast=True)
             sendUSStatItems()
-            logger.debug('LUA ERROR: ' + str(e).replace("\033", ""))
+            logger.error('LUA ERROR: ' + str(e).replace("\033", ""))
             logger.warning("Lua engine stopped; please open 'Userscripts' and press Load to reinitialize scripts.")
         else:
             emit('from_server', {'cmd': 'errmsg', 'data': 'Error occurred during generator call; please check console.'}, broadcast=True)

colab/GPU.ipynb

@@ -70,10 +70,18 @@
         "Model = \"Nerys 2.7B\" #@param [\"Nerys 2.7B\", \"AID 2.7B\", \"Erebus 2.7B\", \"Janeway 2.7B\", \"Picard 2.7B\", \"Horni LN 2.7B\", \"Horni 2.7B\", \"Shinen 2.7B\", \"OPT 2.7B\", \"Fairseq Dense 2.7B\", \"Neo 2.7B\"] {allow-input: true}\n",
         "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
         "Provider = \"Localtunnel\" #@param [\"Localtunnel\", \"Cloudflare\"]\n",
+		"use_google_drive = True #@param {type:\"boolean\"}\n",
         "\n",
         "!nvidia-smi\n",
         "from google.colab import drive\n",
-        "drive.mount('/content/drive/')\n",
+        "if use_google_drive:\n",
+		"  drive.mount('/content/drive/')\n",
+		"else:\n",
+		"  import os\n",
+		"  if not os.path.exists(\"/content/drive\"):\n",
+		"    os.mkdir(\"/content/drive\")\n",
+		"  if not os.path.exists(\"/content/drive/MyDrive/\"):\n",
+		"    os.mkdir(\"/content/drive/MyDrive/\")\n",
         "\n",
         "if Model == \"Nerys 2.7B\":\n",
         "  Model = \"KoboldAI/fairseq-dense-2.7B-Nerys\"\n",

colab/TPU.ipynb

@@ -69,6 +69,7 @@
         "Model = \"Nerys 13B V2\" #@param [\"Nerys 13B V2\", \"Erebus 13B\", \"Janeway 13B\", \"Shinen 13B\", \"Skein 20B\", \"Erebus 20B\", \"Skein 6B\", \"Janeway 6B\", \"Adventure 6B\", \"Shinen 6B\", \"Lit V2 6B\", \"Lit 6B\", \"NeoX 20B\", \"OPT 13B\", \"Fairseq Dense 13B\", \"GPT-J-6B\"] {allow-input: true}\n",
         "Version = \"Official\" #@param [\"Official\", \"United\"] {allow-input: true}\n",
         "Provider = \"Localtunnel\" #@param [\"Localtunnel\", \"Cloudflare\"]\n",
+		"use_google_drive = True #@param {type:\"boolean\"}\n",
         "\n",
         "import os\n",
         "try:\n",
@@ -79,7 +80,14 @@
         "    raise RuntimeError(\"⚠️You can not run this notebook without the TPU accelerator, go to Runtime->Sessions, terminate your session and then try again.⚠️\")\n",
         "print('Now we will need your Google Drive to store settings and saves, you must login with the same account you used for Colab.')\n",
         "from google.colab import drive\n",
-        "drive.mount('/content/drive/')\n",
+        "if use_google_drive:\n",
+		"  drive.mount('/content/drive/')\n",
+		"else:\n",
+		"  import os\n",
+		"  if not os.path.exists(\"/content/drive\"):\n",
+		"    os.mkdir(\"/content/drive\")\n",
+		"  if not os.path.exists(\"/content/drive/MyDrive/\"):\n",
+		"    os.mkdir(\"/content/drive/MyDrive/\")\n",
         "\n",
         "if Model == \"Janeway 13B\":\n",
         "  Model = \"KoboldAI/fairseq-dense-13B-Janeway\"\n",

commandline.bat

@@ -1,5 +1,7 @@
 @echo off
 cd /D %~dp0
+SET CONDA_SHLVL=
+
 TITLE CMD for KoboldAI Runtime
 SET /P M=<loader.settings
 IF %M%==1 GOTO drivemap

environments/finetuneanon.yml

@@ -20,10 +20,13 @@ dependencies:
   - marshmallow>=3.13
   - apispec-webframeworks
   - loguru
+  - termcolor
+  - psutil
   - pip:
     - flask-cloudflared
     - flask-ngrok
     - lupa==1.10
     - transformers>=4.20.1
     - huggingface_hub>=0.10.1
     - accelerate
+    - git+https://github.com/VE-FORBRYDERNE/mkultra

environments/huggingface.yml

@@ -17,6 +17,8 @@ dependencies:
   - marshmallow>=3.13
   - apispec-webframeworks
   - loguru
+  - termcolor
+  - psutil
   - pip:
     - --extra-index-url https://download.pytorch.org/whl/rocm5.1.1
     - torch
@@ -27,3 +29,4 @@ dependencies:
     - transformers>=4.20.1
     - huggingface_hub>=0.10.1
     - accelerate
+    - git+https://github.com/VE-FORBRYDERNE/mkultra

environments/rocm-finetune.yml

@@ -86,7 +86,7 @@ def uspath(filename):
 def getstoryfiles():
     list = []
     for file in listdir("stories"):
-        if file.endswith(".json"):
+        if file.endswith(".json") and not file.endswith(".v2.json"):
             ob = {}
             ob["name"] = file.replace(".json", "")
             f = open("stories/"+file, "r")

environments/rocm.yml

@@ -8,6 +8,7 @@ echo.
 
 Reg add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v "LongPathsEnabled" /t REG_DWORD /d "1" /f 2>nul
 cd /D %~dp0
+SET CONDA_SHLVL=
 
 if exist miniconda3\ (
   echo Delete existing installation?