fix: prevent asset dedup key from collapsing distinct IPs under the same domain

docs/en-US/api-recipes.md

@@ -198,5 +198,5 @@ Notes:
 - The caller needs `asset:write`.
 - Each asset requires at least one of `host`, `ip`, or `domain`.
 - One request supports up to 100,000 assets. For large payloads, use a file with `--data-binary` instead of embedding JSON in the command line.
-- An existing “target + port + protocol” is merged and counted in `updated`.
+- An existing “IP + domain + port + protocol” (or host when both IP and domain are missing) is merged and counted in `updated`.
 - To work from XLSX/CSV, use **Asset Inventory → Bulk Import** in the Web UI. The API itself accepts JSON rather than multipart files.

docs/en-US/api-reference.md

@@ -165,7 +165,7 @@ Request rules:
 - an asset may have up to 30 tags, each no longer than 64 characters;
 - a non-empty `project_id` must reference a project accessible to the caller;
 - the caller needs `asset:write`;
-- the server deduplicates by “target + port + protocol” and processes the request in one transaction.
+- the server deduplicates by “IP + domain + port + protocol” (or host when both IP and domain are missing) and processes the request in one transaction.
 
 Successful response:
 

docs/en-US/asset-management.md

@@ -89,7 +89,7 @@ Limits and behavior:
 
 - One XLSX/CSV file may contain up to 100,000 rows and be up to 100 MB.
 - One `/api/assets/import` request may contain up to 100,000 assets.
-- Later rows with the same “target + port + protocol” in one file are marked as duplicates and are not submitted.
+- Later rows with the same “IP + domain + port + protocol” (or host when both IP and domain are missing) in one file are marked as duplicates and are not submitted.
 - The Web UI parses and previews the file; the server remains responsible for authorization, validation, normalization, deduplication, and transactional writes.
 - Existing assets receive non-empty incoming fields and a refreshed last-seen time instead of a duplicate record.
 - Bulk import requires `asset:write`. Referenced projects must also be accessible to the current user.
@@ -118,7 +118,7 @@ Different sources may describe the same target in different forms. The system:
 - removes empty or duplicate tags;
 - supplies default source and status values.
 
-Assets use “target + port + protocol” as the service-level deduplication key. The preferred target is the domain, followed by the IP address, then the host. As a result, `80/http` and `443/https` on the same host remain separate assets.
+Assets use “IP + domain + port + protocol” as the service-level deduplication key; the host is only used when both IP and domain are missing. As a result, assets under the same domain with different IP addresses (for example, reconnaissance-platform exports where the domain column holds the registrable root domain and the host column holds subdomains) remain separate assets, and `80/http` and `443/https` on the same host also remain separate assets.
 
 When an existing asset is imported again, non-empty incoming fields and the last-seen time are updated. Existing fields omitted by the new record and the original first-seen time are preserved.
 

docs/zh-CN/api-recipes.md

@@ -198,5 +198,5 @@ curl -k https://127.0.0.1:8080/api/assets/import \
 - 调用者需要 `asset:write` 权限。
 - 每条资产至少填写 `host`、`ip` 或 `domain`。
 - 单次最多 100000 条；大批量请求建议使用文件配合 `--data-binary`，不要把 JSON 直接写进命令行。
-- 已存在的“目标 + 端口 + 协议”会合并更新并计入 `updated`。
+- 已存在的“IP + 域名 + 端口 + 协议”（IP 与域名都缺失时为 Host）会合并更新并计入 `updated`。
 - 如需从 XLSX/CSV 操作，使用 Web 端 **资产库 → 批量导入**；接口本身接收 JSON，不接收 multipart 文件。

docs/zh-CN/api-reference.md

@@ -199,7 +199,7 @@ Content-Type: application/json
 - 标签最多 30 个，单个最多 64 个字符；
 - `project_id` 非空时，调用者必须有权访问该项目；
 - 需要 `asset:write` 权限；
-- 服务端按“目标 + 端口 + 协议”去重，并在同一事务中处理本次请求。
+- 服务端按“IP + 域名 + 端口 + 协议”（IP 与域名都缺失时为 Host）去重，并在同一事务中处理本次请求。
 
 成功响应：
 

docs/zh-CN/asset-management.md

@@ -89,7 +89,7 @@ example.com
 
 - 单个 XLSX/CSV 文件最多 100000 行、100 MB。
 - `/api/assets/import` 单次请求最多 100000 条资产。
-- 文件内相同“目标 + 端口 + 协议”的后续行会标记为重复，不会提交。
+- 文件内相同“IP + 域名 + 端口 + 协议”（IP 与域名都缺失时为 Host）的后续行会标记为重复，不会提交。
 - Web 端负责文件解析、预览和即时格式提示；服务端仍会执行权限检查、字段校验、规范化、去重和事务写入。
 - 已存在的资产会合并本次提供的非空字段并更新最近发现时间；不会创建重复记录。
 - 批量导入需要 `asset:write` 权限。填写项目时，当前用户还必须有权访问该项目。
@@ -118,7 +118,7 @@ example.com
 - 清理并去重标签；
 - 为缺失的状态和来源补充默认值。
 
-资产以“目标 + 端口 + 协议”作为服务级去重依据。目标优先采用域名，其次为 IP，最后为 Host。因此，同一主机的 `80/http` 和 `443/https` 会被视为不同资产。
+资产以“IP + 域名 + 端口 + 协议”作为服务级去重依据；IP 与域名都缺失时才回退到 Host 去重。因此，同一域名下不同 IP 的资产（如测绘平台导出中 domain 为根域名、host 为子域名的数据）会被视为不同资产，同一主机的 `80/http` 和 `443/https` 也会被视为不同资产。
 
 重复资产再次入库时，系统会更新本次提供的非空字段和最近发现时间，保留未提供的原有信息，不会重置首次发现时间。
 

internal/app/asset_tools.go

@@ -26,7 +26,7 @@ func registerAssetTools(server *mcp.Server, db *database.DB, logger *zap.Logger)
 
 	server.RegisterTool(mcp.Tool{
 		Name: builtin.ToolCreateAsset, ShortDescription: "新增或去重更新资产",
-		Description: "向资产库新增资产。按目标+端口+协议去重；若资产已存在则更新非空字段。至少提供 host、ip、domain 之一。",
+		Description: "向资产库新增资产。按 IP+域名+端口+协议去重（IP 与域名都缺失时才按 host 去重）；若资产已存在则更新非空字段。至少提供 host、ip、domain 之一。",
 		// Bedrock rejects tool schemas with top-level oneOf/allOf/anyOf. The
 		// host/ip/domain requirement is enforced by assetFromCreateArgs below.
 		InputSchema: map[string]interface{}{"type": "object", "properties": properties},

internal/database/asset.go

@@ -272,15 +272,28 @@ func validAssetDomain(domain string) bool {
 	return true
 }
 
+// assetDedupKey derives asset identity from ip+domain combined: assets with
+// different IPs under the same domain (e.g. reconnaissance exports where the
+// domain column holds the registrable root domain and host holds subdomains)
+// must not be merged. Host only participates as a fallback (pure-URL assets)
+// when both ip and domain are missing, keeping free-form host text out of the
+// key to avoid format sensitivity.
 func assetDedupKey(a *Asset) string {
-	target := a.Domain
-	if target == "" {
-		target = a.IP
+	host := ""
+	if a.IP == "" && a.Domain == "" {
+		host = strings.ToLower(a.Host)
 	}
-	if target == "" {
-		target = strings.ToLower(a.Host)
-	}
-	return strings.Join([]string{target, strconv.Itoa(a.Port), a.Protocol}, "|")
+	return strings.Join([]string{host, a.IP, a.Domain, strconv.Itoa(a.Port), a.Protocol}, "|")
+}
+
+// translateAssetDedupConflict converts a dedup_key UNIQUE constraint conflict
+// into a readable validation error. Under the ip+domain combined identity,
+// editing an asset's identifiers to exactly match an existing asset triggers it.
+func translateAssetDedupConflict(err error) error {
+	if err != nil && strings.Contains(strings.ToLower(err.Error()), "unique constraint failed: assets.dedup_key") {
+		return assetValidationErrorf("目标标识与已有资产冲突，请先合并重复资产或修改 IP/域名/端口/协议")
+	}
+	return err
 }
 
 func appendAssetAccess(query string, args []interface{}, access RBACListAccess, alias string) (string, []interface{}) {
@@ -319,10 +332,6 @@ func (db *DB) UpsertAssets(assets []*Asset, ownerUserID string, allowGlobal ...b
 			return result, fmt.Errorf("第 %d 个资产无效: %w", result.Created+result.Updated+result.Skipped+1, err)
 		}
 		key := assetDedupKey(asset)
-		if key == "|0|" {
-			result.Skipped++
-			continue
-		}
 		var existingID string
 		var existingOwner sql.NullString
 		err := tx.QueryRow(`SELECT id,owner_user_id FROM assets WHERE dedup_key = ?`, key).Scan(&existingID, &existingOwner)
@@ -888,17 +897,14 @@ func (db *DB) UpdateAsset(id string, a *Asset, access RBACListAccess) error {
 		return err
 	}
 	key := assetDedupKey(a)
-	if key == "|0|" {
-		return fmt.Errorf("资产目标不能为空")
-	}
 	tags, _ := json.Marshal(a.Tags)
 	where, args := appendAssetAccess(" WHERE id = ?", []interface{}{id}, access, "assets")
 	res, err := db.Exec(`UPDATE assets SET dedup_key=?,project_id=?,host=?,ip=?,port=?,domain=?,protocol=?,title=?,server=?,country=?,province=?,city=?,
 		responsible_person=?,department=?,business_system=?,environment=?,criticality=?,source=?,source_query=?,status=?,tags_json=?,updated_at=?`+where,
 		append([]interface{}{key, nullIfEmpty(a.ProjectID), a.Host, a.IP, a.Port, a.Domain, a.Protocol, a.Title, a.Server, a.Country, a.Province, a.City,
 			a.ResponsiblePerson, a.Department, a.BusinessSystem, a.Environment, a.Criticality, a.Source, a.SourceQuery, a.Status, string(tags), time.Now()}, args...)...)
 	if err != nil {
-		return err
+		return translateAssetDedupConflict(err)
 	}
 	n, _ := res.RowsAffected()
 	if n == 0 {
@@ -1163,7 +1169,7 @@ func (db *DB) MergeAssets(primary *Asset, duplicateIDs []string, writeAccess, de
 			primary.Criticality, primary.Source, primary.SourceQuery, primary.Status, string(tagsJSON), time.Now(), primary.ID}, writeAccess, "assets")
 	result, err := tx.Exec(updateQuery, updateScopeArgs...)
 	if err != nil {
-		return 0, err
+		return 0, translateAssetDedupConflict(err)
 	}
 	if updated, _ := result.RowsAffected(); updated != 1 {
 		return 0, fmt.Errorf("更新主资产失败")

internal/database/asset_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"errors"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -128,6 +129,110 @@ func TestAssetUpsertDeduplicatesAndUpdates(t *testing.T) {
 	}
 }
 
+// TestAssetUpsertKeepsDistinctIPsUnderSameDomain is a regression test: in
+// reconnaissance exports the domain column often holds the registrable root
+// domain while host holds subdomains, so assets with different IPs under the
+// same domain must not be merged by deduplication.
+func TestAssetUpsertKeepsDistinctIPsUnderSameDomain(t *testing.T) {
+	db, err := NewDB(filepath.Join(t.TempDir(), "assets-dedup-ip.db"), zap.NewNop())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	result, err := db.UpsertAssets([]*Asset{
+		{Host: "https://a.example.com:3000", IP: "192.0.2.1", Domain: "example.com", Port: 3000, Protocol: "https", Source: "fofa"},
+		{Host: "https://b.example.com:3000", IP: "192.0.2.2", Domain: "example.com", Port: 3000, Protocol: "https", Source: "fofa"},
+	}, "")
+	if err != nil || result.Created != 2 {
+		t.Fatalf("distinct IPs under same domain were merged: %#v, %v", result, err)
+	}
+	// Rows with identical fields must still hit dedup and update in place.
+	result, err = db.UpsertAssets([]*Asset{
+		{Host: "https://a.example.com:3000", IP: "192.0.2.1", Domain: "example.com", Port: 3000, Protocol: "https", Title: "New", Source: "fofa"},
+	}, "")
+	if err != nil || result.Created != 0 || result.Updated != 1 {
+		t.Fatalf("identical row not deduplicated: %#v, %v", result, err)
+	}
+	_, total, err := db.ListAssets(20, 0, AssetListFilter{}, RBACListAccess{Scope: RBACScopeAll})
+	if err != nil || total != 2 {
+		t.Fatalf("assets total=%d err=%v", total, err)
+	}
+}
+
+// TestAssetDedupKeyMigrationRecomputesLegacyKeys verifies that legacy-format
+// dedup keys are recomputed by migrateAssetsTable on startup (covers the
+// upgrade.sh scenario where data/ is preserved across upgrades).
+func TestAssetDedupKeyMigrationRecomputesLegacyKeys(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "assets-dedup-migrate.db")
+	db, err := NewDB(path, zap.NewNop())
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Now()
+	// Insert a legacy-format (domain-priority) dedup key directly to simulate
+	// pre-upgrade data.
+	if _, err := db.Exec(`INSERT INTO assets (id,dedup_key,host,ip,port,domain,protocol,first_seen_at,last_seen_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
+		"asset-legacy", "example.com|80|http", "a.example.com", "192.0.2.1", 80, "example.com", "http", now, now, now, now); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db, err = NewDB(path, zap.NewNop())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	var key string
+	if err := db.QueryRow(`SELECT dedup_key FROM assets WHERE id='asset-legacy'`).Scan(&key); err != nil {
+		t.Fatal(err)
+	}
+	if key != "|192.0.2.1|example.com|80|http" {
+		t.Fatalf("legacy dedup key not migrated: %q", key)
+	}
+}
+
+// TestUpdateAssetRejectsDedupKeyConflict verifies that editing an asset's
+// identifiers to exactly match a coexisting asset turns the UNIQUE conflict
+// into a readable validation error instead of the raw SQLite message.
+func TestUpdateAssetRejectsDedupKeyConflict(t *testing.T) {
+	db, err := NewDB(filepath.Join(t.TempDir(), "assets-dedup-conflict.db"), zap.NewNop())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	result, err := db.UpsertAssets([]*Asset{
+		{IP: "192.0.2.1", Domain: "example.com", Port: 80, Protocol: "http"},
+		{IP: "192.0.2.2", Domain: "example.com", Port: 80, Protocol: "http"},
+	}, "")
+	if err != nil || result.Created != 2 {
+		t.Fatalf("setup upsert = %#v, %v", result, err)
+	}
+	assets, _, err := db.ListAssets(20, 0, AssetListFilter{}, RBACListAccess{Scope: RBACScopeAll})
+	if err != nil || len(assets) != 2 {
+		t.Fatalf("setup list err=%v len=%d", err, len(assets))
+	}
+	var targetID string
+	for _, asset := range assets {
+		if asset.IP == "192.0.2.2" {
+			targetID = asset.ID
+		}
+	}
+	if targetID == "" {
+		t.Fatal("setup asset not found")
+	}
+	err = db.UpdateAsset(targetID, &Asset{IP: "192.0.2.1", Domain: "example.com", Port: 80, Protocol: "http"}, RBACListAccess{Scope: RBACScopeAll})
+	if err == nil {
+		t.Fatal("expected dedup key conflict error")
+	}
+	var validationErr *AssetValidationError
+	if !errors.As(err, &validationErr) {
+		t.Fatalf("conflict not translated to validation error: %v", err)
+	}
+}
+
 func TestAssetAccessFiltersOwners(t *testing.T) {
 	db, err := NewDB(filepath.Join(t.TempDir(), "assets-access.db"), zap.NewNop())
 	if err != nil {

internal/database/database.go

@@ -1045,7 +1045,67 @@ func (db *DB) migrateAssetsTable() error {
 			}
 		}
 	}
-	return nil
+	// 去重键语义升级：旧版按 domain>ip>host 优先级单标识符去重，会把同域名下不同 IP
+	// 的资产误判为重复；新版改为 ip+domain 联合标识（host 兜底）。存量键重算是幂等的，
+	// 且新键区分度更细，不可能出现两条存量记录重算后撞 UNIQUE 约束的情况。
+	// 快速探测：所有行都已符合新格式键时跳过全表重算（迁移完成后的每次启动只需
+	// 付这一次轻量 SQL 扫描）。探测用值比对而非数 '|' 分隔符：host 为自由文本可能
+	// 含 '|'，数分隔符会漏判。normalizeAsset 保证 ip/domain/protocol 写入时已小写，
+	// 探测不会漏检；极端情况（host 兜底键含非 ASCII 大写字母）可能误报，仅导致
+	// 一次无写入的空转重算。
+	probe := `(CASE WHEN ip = '' AND domain = '' THEN lower(host) ELSE '' END) || '|' || ip || '|' || domain || '|' || port || '|' || protocol`
+	var needsMigration bool
+	if err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM assets WHERE dedup_key <> (` + probe + `))`).Scan(&needsMigration); err != nil {
+		return err
+	}
+	if !needsMigration {
+		return nil
+	}
+	// 重算在 Go 中进行以与 assetDedupKey 语义完全一致（SQLite 内置 lower() 仅处理
+	// ASCII，host 兜底键在含非 ASCII 大写字母时会与 Go 的 Unicode 小写化产生偏差）。
+	rows, err := db.Query(`SELECT id, dedup_key, host, ip, port, domain, protocol FROM assets`)
+	if err != nil {
+		return err
+	}
+	type dedupKeyUpdate struct {
+		id  string
+		key string
+	}
+	var updates []dedupKeyUpdate
+	for rows.Next() {
+		var id, oldKey string
+		var asset Asset
+		if err := rows.Scan(&id, &oldKey, &asset.Host, &asset.IP, &asset.Port, &asset.Domain, &asset.Protocol); err != nil {
+			rows.Close()
+			return err
+		}
+		asset.IP = strings.ToLower(asset.IP)
+		asset.Domain = strings.ToLower(asset.Domain)
+		asset.Protocol = strings.ToLower(asset.Protocol)
+		if key := assetDedupKey(&asset); key != oldKey {
+			updates = append(updates, dedupKeyUpdate{id: id, key: key})
+		}
+	}
+	if err := rows.Err(); err != nil {
+		rows.Close()
+		return err
+	}
+	rows.Close()
+	if len(updates) == 0 {
+		return nil
+	}
+	// 单事务批量写回，避免逐行自动提交在首次升级（全量行都需重算）时线性放大启动耗时。
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	for _, update := range updates {
+		if _, err := tx.Exec(`UPDATE assets SET dedup_key = ? WHERE id = ?`, update.key, update.id); err != nil {
+			return err
+		}
+	}
+	return tx.Commit()
 }
 
 // migrateMessagesTable 迁移 messages 表，补充 updated_at 字段。

internal/handler/openapi.go

@@ -2603,7 +2603,7 @@ func (h *OpenAPIHandler) GetOpenAPISpec(c *gin.Context) {
 				"post": map[string]interface{}{
 					"tags":        []string{"资产管理"},
 					"summary":     "批量导入资产",
-					"description": "新增或按“目标 + 端口 + 协议”去重更新资产。接收 JSON，不直接接收 XLSX/CSV 文件；单次最多 100000 条，需要 asset:write 权限。",
+					"description": "新增或按“IP + 域名 + 端口 + 协议”（IP 与域名都缺失时为 Host）去重更新资产。接收 JSON，不直接接收 XLSX/CSV 文件；单次最多 100000 条，需要 asset:write 权限。",
 					"operationId": "importAssets",
 					"requestBody": map[string]interface{}{
 						"required": true,

web/static/js/assets.js

@@ -272,7 +272,9 @@ function parseAssetImportMatrix(matrix, fileName) {
         const parsed = assetImportRecord(values, index + 2);
         if (!parsed.error) {
             const asset = parsed.asset;
-            const key = `${String(asset.domain || asset.ip || asset.host).toLowerCase()}|${asset.port || 0}|${asset.protocol || ''}`;
+            // 与后端 assetDedupKey 保持一致：ip+domain 联合标识，host 仅在两者都缺失时兜底。
+            const hostKey = (!asset.ip && !asset.domain) ? String(asset.host || '').toLowerCase() : '';
+            const key = `${hostKey}|${asset.ip || ''}|${asset.domain || ''}|${asset.port || 0}|${asset.protocol || ''}`;
             if (seen.has(key)) parsed.error = assetT('assets.duplicateFileRow', `与第 ${seen.get(key)} 行重复`, { row: seen.get(key) });
             else seen.set(key, parsed.rowNumber);
         }