From 8febfecf7b877fc062aa3142384962cb4634c16f Mon Sep 17 00:00:00 2001 From: Samuel Yew <135582318+cs168898@users.noreply.github.com> Date: Wed, 7 Jan 2026 03:08:19 +0000 Subject: [PATCH 1/2] feature: Added drop duplicates feature added drop duplicates feature where user can drop any duplicates in the dataframe, options to pass into the method can be used, such as selecting custom column names to drop duplicates, Keep either the first, last or none of the rows and whether to modify the original dataframe or create a another dataframe with the modified rows --- dataframe/cleaning.go | 191 +++++++++++++++++++++++++++++++++++++++++- goframe.go | 1 + 2 files changed, 191 insertions(+), 1 deletion(-) diff --git a/dataframe/cleaning.go b/dataframe/cleaning.go index 4304b23..c8c27e9 100644 --- a/dataframe/cleaning.go +++ b/dataframe/cleaning.go @@ -1,6 +1,10 @@ package dataframe -import "fmt" +import ( + "fmt" + "sort" + "strings" +) // Data Cleaning @@ -79,3 +83,188 @@ func (df *DataFrame) Astype(columnName string, targetType string) error { col.Data = newData return nil } + +// DropDuplicatesOption is the parameters we can set to the DropDuplicates method. +// +// Fields: +// - subset: The columns to identify duplicates +// - keep: Determines which duplicates to keep, "first" keeps the first occurence, +// "last" keeps the last occurence and "none" drops every duplicate value's row. +// - inplace: Determines if the original DataFrame should be modified. +type DropDuplicatesOption struct { + Subset []string // The column names to drop duplicates + Keep string //"first", "last", "none" + Inplace bool // To return the current modified dataframe or cloned + // ignoreIndex bool ---> This could be added later on, to re organise the index (Refer to pandas DropDuplicate method) +} + +// DropDuplicates is the method where users can drop duplicate values in the dataframe. +// It will return a cloned DataFrame by default and only keep the FIRST occurence of the duplicate value. +// +// Parameters: +// - options: The DropDuplicatesOption struct to optionally add parameters to this method. +// +// Returns: +// - *DataFrame: The original dataframe with no duplicates if the inplace option in DropDuplicatesOption is true. +// - *DataFrame (cloned): A cloned DataFrame if the inplace option in DropDuplicatesOption is false. +func (df *DataFrame) DropDuplicates(options ...DropDuplicatesOption) (*DataFrame, error) { + var colNames []string + finalOptions := DropDuplicatesOption{Keep: "first", Inplace: false} + newDf := NewDataFrame() + + if len(options) > 0 { + userOpt := options[0] + + // only overwrite Subset if user provided one + if len(userOpt.Subset) > 0 { + finalOptions.Subset = userOpt.Subset + } + + // only overwrite Keep if user provided one (not empty) + if userOpt.Keep != "" { + finalOptions.Keep = userOpt.Keep + } + + finalOptions.Inplace = userOpt.Inplace + } + + if len(finalOptions.Subset) > 0 { + // if the options has columns to specifically target + colNames = finalOptions.Subset + } else { + // else just do it for all columns + colNames = df.ColumnNames() + } + + // find which indexes in a column to keep (which rows to keep), prob create another helper function to do this + seen := make(map[string]bool) + indexesToKeep := []int{} + + switch finalOptions.Keep { + case "first": + // logic for finalOptions.keep = "first" + for i := 0; i < df.Nrows(); i++ { + rowKey, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if !seen[rowKey] { + seen[rowKey] = true + indexesToKeep = append(indexesToKeep, i) + } + } + + case "last": + // logic for finalOptions.keep = "last" + for i := df.Nrows() - 1; i >= 0; i-- { + rowKey, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if !seen[rowKey] { + seen[rowKey] = true + indexesToKeep = append(indexesToKeep, i) + } + } + sort.Ints(indexesToKeep) + + case "none": + // logic for finalOptions.keep = "none" + + // count all the times a specific row has occured before + counts := make(map[string]int) + for i := 0; i < df.Nrows(); i++ { + key, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + counts[key]++ + } + + // only keep the indexes where the count == 1 + for i := 0; i < df.Nrows(); i++ { + key, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if counts[key] == 1 { + indexesToKeep = append(indexesToKeep, i) + } + } + + } + + for _, colName := range df.ColumnNames() { + // iterate through each column and get the indexes + // then directly change it in the current dataframe + // or add the new data to a new dataframe to return. + newData, err := df.getSubSlice(colName, indexesToKeep) + + if err != nil { + return nil, fmt.Errorf("Error, could not get sub slice: %v", err) + } + + // use inplace option to check whether we want to clone or modify the current dataframe + if finalOptions.Inplace { + // replace the existing data with the new data + df.Columns[colName].Data = newData + } else { + err := newDf.AddColumn(ConvertToAnyColumn(NewColumn(colName, newData))) + if err != nil { + return nil, fmt.Errorf("Error trying to add column to new DataFrame: %v", err) + } + } + } + + if finalOptions.Inplace { + return df, nil + } + + return newDf, nil + // return the DataFrame +} + +// getRowKey is a helper function to return a unique key for a row +func (df *DataFrame) getRowKey(rowIndex int, colNames []string) (string, error) { + + var builder strings.Builder + + for _, name := range colNames { + col, ok := df.Columns[name] + if !ok { + return "", fmt.Errorf("Column %s not found", name) + } + value := col.Data[rowIndex] + + // add the col name to prevent similar values but different column cases + builder.WriteString(name) + builder.WriteString(":") + + if value == nil { + builder.WriteString("nil") + } else { + builder.WriteString(fmt.Sprintf("%v", value)) + } + + builder.WriteString("|") + } + + return builder.String(), nil +} + +// getSubSlice is a method get a portion of an existing column and returns a slice +func (df *DataFrame) getSubSlice(colName string, indexesToKeep []int) ([]any, error) { + col, ok := df.Columns[colName] + if !ok { + return nil, fmt.Errorf("Column %s not found", colName) + } + + finalRows := make([]any, len(indexesToKeep)) + + for i, index := range indexesToKeep { + // use direct assignment instead of appending because appending costs more + finalRows[i] = col.Data[index] + } + + return finalRows, nil +} diff --git a/goframe.go b/goframe.go index ef284fb..a063003 100644 --- a/goframe.go +++ b/goframe.go @@ -17,6 +17,7 @@ type MultiIndex = df.MultiIndex type GroupedDataFrame = df.GroupedDataFrame type DataFrameSorter = df.DataFrameSorter type FuncType = df.FuncType +type DropDuplicatesOption = df.DropDuplicatesOption // Column is re-exported as a generic type alias type Column[T any] = df.Column[T] From 59ef95a6da431c633d295c1af2a592018d0c2216 Mon Sep 17 00:00:00 2001 From: Samuel Yew <135582318+cs168898@users.noreply.github.com> Date: Wed, 7 Jan 2026 03:09:33 +0000 Subject: [PATCH 2/2] test: added DropDuplicates Test added drop duplicates test for all of the options as well as subset and keep in the same option struct --- goframe_tests/dataframe_test.go | 289 ++++++++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) diff --git a/goframe_tests/dataframe_test.go b/goframe_tests/dataframe_test.go index 47e8978..f5b07b3 100644 --- a/goframe_tests/dataframe_test.go +++ b/goframe_tests/dataframe_test.go @@ -1378,6 +1378,295 @@ func TestDataFrameSortValues(t *testing.T) { }) } +// MARK: DropDuplicates +func TestDropDuplicates(t *testing.T) { + // This helper returns a fresh DF every time it is called + setupDF := func() *goframe.DataFrame { + df := goframe.NewDataFrame() + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("name", []string{"Charlie", "Alice", "Alice"}))) + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("age", []int{35, 25, 25}))) + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("score", []float64{90.5, 95.0, 95.0}))) + return df + } + t.Run("DefaultOptionsTest", func(t *testing.T) { + df := setupDF() + finalDf, err := df.DropDuplicates() + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + numberOfRows := finalDf.Nrows() + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i, val := range scoreCol.Data { + if val.(float64) != expectedScores[i] { + t.Errorf("Expected score at index %d to be %v, got %v", i, expectedScores[i], val) + } + } + + for i, val := range nameCol.Data { + if val.(string) != expectedNames[i] { + t.Errorf("Expected name at index %d to be %s, got %v", i, expectedNames[i], val) + } + } + + for i, val := range ageCol.Data { + if val.(int) != expectedAge[i] { + t.Errorf("Expected age at index %d to be %v, got %v", i, expectedAge[i], val) + } + } + }) + + t.Run("Subset Option", func(t *testing.T) { + + df2 := goframe.NewDataFrame() + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("name", []string{"Charlie", "Alice", "Bob"}))) + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("age", []int{35, 25, 25}))) // age is the same for both Alice and Bob + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("score", []float64{90.5, 95.0, 80.0}))) + + dropDuplicateOption := goframe.DropDuplicatesOption{ + Subset: []string{"age"}, + } + finalDf, err := df2.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + numberOfRows := finalDf.Nrows() + + if numberOfRows != 2 { + t.Errorf("Expected 1 row but got %v", numberOfRows) + } + + if len(scoreCol.Data) != len(expectedScores) { + t.Fatalf("Column length mismatch: got %d, want %d", len(scoreCol.Data), len(expectedScores)) + } + + for i, val := range scoreCol.Data { + if val.(float64) != expectedScores[i] { + t.Errorf("Expected score at index %d to be %v, got %v", i, expectedScores[i], val) + } + } + + for i, val := range nameCol.Data { + if val.(string) != expectedNames[i] { + t.Errorf("Expected name at index %d to be %s, got %v", i, expectedNames[i], val) + } + } + + for i, val := range ageCol.Data { + if val.(int) != expectedAge[i] { + t.Errorf("Expected age at index %d to be %v, got %v", i, expectedAge[i], val) + } + } + }) + + // + t.Run("KeepOptions", func(t *testing.T) { + df := setupDF() + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("index", []int{0, 1, 2}))) + t.Run("KeepFirst", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "first", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + expectedIndex := []int{0, 1} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 2 + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + + t.Run("KeepLast", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "last", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + expectedIndex := []int{0, 2} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 2 + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + + t.Run("KeepNone", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "none", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie"} + expectedAge := []int{35} + expectedScores := []float64{90.5} + expectedIndex := []int{0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 1 + + if numberOfRows != 1 { + t.Errorf("Expected 1 row but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + }) + + t.Run("InplaceTrue_ModifiesOriginal", func(t *testing.T) { + df := setupDF() + + _, err := df.DropDuplicates(goframe.DropDuplicatesOption{Inplace: true}) + if err != nil { + t.Fatalf("Operation failed: %v", err) + } + + // check 1: the original 'df' variable must now have fewer rows + expectedRows := 2 + if df.Nrows() != expectedRows { + t.Errorf("Expected original DF to be modified to %d rows, but got %d", expectedRows, df.Nrows()) + } + + // check 2: verify the data inside the original 'df' is the cleaned data + nameCol, _ := df.Select("name") + if len(nameCol.Data) != expectedRows { + t.Errorf("Original column data length mismatch: expected %d, got %d", expectedRows, len(nameCol.Data)) + } + + }) + + t.Run("InplaceFalse_PreservesOriginal", func(t *testing.T) { + df := setupDF() // Start with 3 rows + + newDf, err := df.DropDuplicates(goframe.DropDuplicatesOption{Inplace: false}) + if err != nil { + t.Fatalf("Operation failed: %v", err) + } + + // check 1: the new DataFrame should be cleaned (2 rows) + if newDf.Nrows() != 2 { + t.Errorf("New DataFrame expected 2 rows, got %d", newDf.Nrows()) + } + + // check 2: the original DataFrame MUST still have 3 rows + if df.Nrows() != 3 { + t.Errorf("Original DataFrame was accidentally modified! Expected 3 rows, got %d", df.Nrows()) + } + }) + +} + // MARK: Helper Functions /*