Skip to content

Commit c08d22c

Browse files
KeyangKeyang
committed
fix #158 
1 parent 68e2d7e commit c08d22c

File tree

5 files changed

+128
-25
lines changed

5 files changed

+128
-25
lines changed

‎libs/core/Converter.js‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ var fileLineToCSVLine = require("./fileLineToCSVLine");
99
var linesToJson = require("./linesToJson");
1010
var CSVError = require("./CSVError");
1111
var workerMgr = null;
12+
var _ = require('lodash');
13+
var rowSplit = require("./rowSplit");
1214
function Converter(params, options) {
1315
Transform.call(this, options);
1416
this._options = options || {};
@@ -212,33 +214,88 @@ Converter.prototype.processHead = function (fileLine, cb) {
212214
if (params._headers) {
213215
return cb();
214216
}
215-
217+
//dirty hack
218+
params._needFilterRow = false;
216219
// if header is not inited. init header
217-
var lines = fileLineToCSVLine(fileLine, params);
218-
this.setPartialData(lines.partial);
220+
var lines = fileLine.lines;
221+
var left = "";
222+
var headerRow = [];
223+
if (!params.noheader) {
224+
while (lines.length) {
225+
var line = left + lines.shift();
226+
var row = rowSplit(line, params);
227+
if (row.closed) {
228+
headerRow = row.cols;
229+
left = "";
230+
break;
231+
} else {
232+
left = line + this.getEol();
233+
}
234+
}
235+
}
236+
params._needFilterRow = true;
237+
if (!params.noheader && headerRow.length === 0) { //if one chunk of data does not complete header row.
238+
this.setPartialData(left);
239+
return cb();
240+
}
219241
if (params.noheader) {
220242
if (params.headers) {
221243
params._headers = params.headers;
222244
} else {
223245
params._headers = [];
224246
}
225247
} else {
226-
var headerRow = lines.lines.shift();
227248
if (params.headers) {
228249
params._headers = params.headers;
229250
} else {
230251
params._headers = headerRow;
231252
}
232253
}
254+
configIgnoreIncludeColumns(params);
255+
params._headers = require("./filterRow")(params._headers, params);
256+
var lines = fileLineToCSVLine(fileLine, params);
257+
this.setPartialData(lines.partial);
233258
if (this.param.workerNum > 1) {
234259
this.workerMgr.setParams(params);
235260
}
236261
var res = linesToJson(lines.lines, params, 0);
237262
this.processResult(res);
238263
this.lastIndex += res.length;
239264
this.recordNum += res.length;
265+
240266
cb();
241267
};
268+
function configIgnoreIncludeColumns(params) {
269+
if (params._postIgnoreColumns) {
270+
for (var i = 0; i < params.ignoreColumns.length; i++) {
271+
var ignoreCol = params.ignoreColumns[i];
272+
if (typeof ignoreCol === "string") {
273+
var idx = params._headers.indexOf(ignoreCol);
274+
if (idx > -1) {
275+
params.ignoreColumns[i] = idx;
276+
} else {
277+
params.ignoreColumns[i] = -1;
278+
}
279+
}
280+
}
281+
params.ignoreColumns.sort(function (a, b) { return b - a; });
282+
}
283+
if (params._postIncludeColumns) {
284+
for (var i = 0; i < params.includeColumns.length; i++) {
285+
var includeCol = params.includeColumns[i];
286+
if (typeof includeCol === "string") {
287+
var idx = params._headers.indexOf(includeCol);
288+
if (idx > -1) {
289+
params.includeColumns[i] = idx;
290+
} else {
291+
params.includeColumns[i] = -1;
292+
}
293+
}
294+
}
295+
}
296+
params.ignoreColumns = _.uniq(params.ignoreColumns);
297+
params.includeColumns = _.uniq(params.includeColumns);
298+
}
242299

243300
Converter.prototype.processResult = function (result) {
244301
for (var i = 0, len = result.length; i < len; i++) {

‎libs/core/defParam.js‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
var numExp = /^[0-9]+$/;
12
module.exports = function (params) {
23
var _param = {
34
constructResult: true, //set to false to not construct result in memory. suitable for big csv data
@@ -23,7 +24,7 @@ module.exports = function (params) {
2324
_headerTitle: [],
2425
_headerFlag: [],
2526
_headers: null,
26-
_needFilterRow:false
27+
_needFilterRow: false
2728
};
2829
if (!params) {
2930
params = {};
@@ -33,11 +34,21 @@ module.exports = function (params) {
3334
_param[key] = params[key];
3435
}
3536
}
37+
if (_param.ignoreColumns.length > 0 && !numExp.test(_param.ignoreColumns.join(""))) {
38+
_param._postIgnoreColumns = true;
39+
}
40+
if (_param.includeColumns.length > 0 && !numExp.test(_param.includeColumns.join(""))) {
41+
_param._postIncludeColumns = true;
42+
}
43+
3644
if (_param.ignoreColumns.length || _param.includeColumns.length) {
3745
_param._needFilterRow = true;
38-
_param.ignoreColumns.sort(function (a, b) { return b - a; });
46+
if (!_param._postIgnoreColumns){
47+
_param.ignoreColumns.sort(function (a, b) { return b-a;});
48+
}
3949
}
4050

51+
4152
return _param;
4253
};
4354

‎libs/core/filterRow.js‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
module.exports=function filterRow(row, param) {
2+
if (param.ignoreColumns instanceof Array && param.ignoreColumns.length > 0) {
3+
for (var igRow = 0, igColLen = param.ignoreColumns.length; igRow < igColLen; igRow++) {
4+
if (param.ignoreColumns[igRow] >= 0) {
5+
row.splice(param.ignoreColumns[igRow], 1);
6+
}
7+
}
8+
}
9+
if (param.includeColumns instanceof Array && param.includeColumns.length > 0) {
10+
var cleanRowArr = [];
11+
for (var inRow = 0, inColLen = param.includeColumns.length; inRow < inColLen; inRow++) {
12+
if (param.includeColumns[inRow] >= 0) {
13+
cleanRowArr.push(row[param.includeColumns[inRow]]);
14+
}
15+
}
16+
row = cleanRowArr;
17+
}
18+
return row;
19+
}

‎libs/core/rowSplit.js‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
var getDelimiter = require("./getDelimiter");
2+
var filterRow=require("./filterRow");
23
/**
34
* Convert a line of string to csv columns according to its delimiter
5+
* the param._header may not be ready when this is called.
46
* @param {[type]} rowStr [description]
57
* @param {[type]} param [Converter param]
68
* @return {[type]} {cols:["a","b","c"],closed:boolean} the closed field indicate if the row is a complete row
@@ -82,25 +84,7 @@ module.exports = function rowSplit(rowStr, param) {
8284

8385
};
8486

85-
function filterRow(row, param) {
86-
if (param.ignoreColumns instanceof Array && param.ignoreColumns.length > 0) {
87-
for (var igRow = 0, igColLen = param.ignoreColumns.length; igRow < igColLen; igRow++) {
88-
if (param.ignoreColumns[igRow] >= 0) {
89-
row.splice(param.ignoreColumns[igRow], 1);
90-
}
91-
}
92-
}
93-
if (param.includeColumns instanceof Array && param.includeColumns.length > 0) {
94-
var cleanRowArr = [];
95-
for (var inRow = 0, inColLen = param.includeColumns.length; inRow < inColLen; inRow++) {
96-
if (param.includeColumns[inRow] >= 0) {
97-
cleanRowArr.push(row[param.includeColumns[inRow]]);
98-
}
99-
}
100-
row = cleanRowArr;
101-
}
102-
return row;
103-
}
87+
10488

10589
function isQuoteOpen(str, param) {
10690
var quote = param.quote;

‎test/testCSVConverter3.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
var csv = require("../");
2+
var assert = require("assert");
3+
var fs = require("fs");
4+
describe("CSV Converter", function () {
5+
it ("should ignore column only once",function(done){
6+
csv({
7+
ignoreColumns:[0,0]
8+
})
9+
.fromFile(__dirname+"/data/complexJSONCSV")
10+
.on('json',function(json){
11+
assert(!json.fieldA.title);
12+
assert(json.fieldA.children[0].name);
13+
})
14+
.on('done',function(){
15+
done()
16+
});
17+
})
18+
it ("should ignore column by header name",function(done){
19+
csv({
20+
ignoreColumns:[0,"fieldA.title",2]
21+
})
22+
.fromFile(__dirname+"/data/complexJSONCSV")
23+
.on('json',function(json){
24+
assert(!json.fieldA.title);
25+
assert(json.fieldA.children[0].name);
26+
assert(!json.fieldA.children[0].id);
27+
})
28+
.on('done',function(){
29+
done()
30+
});
31+
})
32+
});

0 commit comments

Comments
 (0)