From 2f61ad3ee82e261f401fc3406c0bc49ab35676b7 Mon Sep 17 00:00:00 2001 From: keighbee Date: Tue, 28 Jan 2025 09:28:48 -0800 Subject: [PATCH 1/5] package.swift test --- Package.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Package.swift b/Package.swift index fc28be1..c4f82c9 100644 --- a/Package.swift +++ b/Package.swift @@ -8,6 +8,7 @@ let package = Package( platforms: [.iOS(.v16), .macOS(.v13)], products: [ .library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]), + .library(name: "Hub", targets: ["Hub"]), .executable(name: "transformers", targets: ["TransformersCLI"]), .executable(name: "hub-cli", targets: ["HubCLI"]), ], From 25bf49dcb43755201becd9379bceb0a4e82a7724 Mon Sep 17 00:00:00 2001 From: keighbee Date: Tue, 28 Jan 2025 11:24:57 -0800 Subject: [PATCH 2/5] added tokenizers as a lib --- Package.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Package.swift b/Package.swift index c4f82c9..a7148e3 100644 --- a/Package.swift +++ b/Package.swift @@ -9,6 +9,7 @@ let package = Package( products: [ .library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]), .library(name: "Hub", targets: ["Hub"]), + .library(name: "Tokenizers", targets: ["Tokenizers"]), .executable(name: "transformers", targets: ["TransformersCLI"]), .executable(name: "hub-cli", targets: ["HubCLI"]), ], From 65b0cadada39d51e21407be9edfd603a557df696 Mon Sep 17 00:00:00 2001 From: keighbee Date: Tue, 28 Jan 2025 11:29:32 -0800 Subject: [PATCH 3/5] added nessisary target to tokenizers --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index a7148e3..edf1776 100644 --- a/Package.swift +++ b/Package.swift @@ -9,7 +9,7 @@ let package = Package( products: [ .library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]), .library(name: "Hub", targets: ["Hub"]), - .library(name: "Tokenizers", targets: ["Tokenizers"]), + .library(name: "Tokenizers", targets: ["Tokenizers", "Models"]), .executable(name: "transformers", targets: ["TransformersCLI"]), .executable(name: "hub-cli", targets: ["HubCLI"]), ], From f1d52336fa0b862d398b4abc00ec1436b16df553 Mon Sep 17 00:00:00 2001 From: keighbee Date: Wed, 29 Jan 2025 11:11:47 -0800 Subject: [PATCH 4/5] removed models as a target from tokenizers library to triage why this breaks --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index edf1776..a7148e3 100644 --- a/Package.swift +++ b/Package.swift @@ -9,7 +9,7 @@ let package = Package( products: [ .library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]), .library(name: "Hub", targets: ["Hub"]), - .library(name: "Tokenizers", targets: ["Tokenizers", "Models"]), + .library(name: "Tokenizers", targets: ["Tokenizers"]), .executable(name: "transformers", targets: ["TransformersCLI"]), .executable(name: "hub-cli", targets: ["HubCLI"]), ], From 1b64c9245e433487ed78d8c6d14177878496079f Mon Sep 17 00:00:00 2001 From: keighbee Date: Wed, 29 Jan 2025 11:44:32 -0800 Subject: [PATCH 5/5] removed TokenizerTests dependency on Model --- Tests/TokenizersTests/TokenizerTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/TokenizersTests/TokenizerTests.swift b/Tests/TokenizersTests/TokenizerTests.swift index eae7003..8862bcd 100644 --- a/Tests/TokenizersTests/TokenizerTests.swift +++ b/Tests/TokenizersTests/TokenizerTests.swift @@ -9,7 +9,6 @@ import XCTest import Hub @testable import Tokenizers -@testable import Models class GPT2TokenizerTests: TokenizerTests { override class var hubModelName: String? { "distilgpt2" } @@ -277,7 +276,8 @@ class TokenizerTester { guard _tokenizer == nil else { return _tokenizer! } do { guard let tokenizerConfig = try await configuration!.tokenizerConfig else { - throw TokenizerError.tokenizerConfigNotFound + XCTFail("Cannot retrieve Tokenizer configuration") + return nil } let tokenizerData = try await configuration!.tokenizerData _tokenizer = try AutoTokenizer.from(tokenizerConfig: tokenizerConfig, tokenizerData: tokenizerData)