diff --git a/dataframe/cleaning.go b/dataframe/cleaning.go index 4304b23..c8c27e9 100644 --- a/dataframe/cleaning.go +++ b/dataframe/cleaning.go @@ -1,6 +1,10 @@ package dataframe -import "fmt" +import ( + "fmt" + "sort" + "strings" +) // Data Cleaning @@ -79,3 +83,188 @@ func (df *DataFrame) Astype(columnName string, targetType string) error { col.Data = newData return nil } + +// DropDuplicatesOption is the parameters we can set to the DropDuplicates method. +// +// Fields: +// - subset: The columns to identify duplicates +// - keep: Determines which duplicates to keep, "first" keeps the first occurence, +// "last" keeps the last occurence and "none" drops every duplicate value's row. +// - inplace: Determines if the original DataFrame should be modified. +type DropDuplicatesOption struct { + Subset []string // The column names to drop duplicates + Keep string //"first", "last", "none" + Inplace bool // To return the current modified dataframe or cloned + // ignoreIndex bool ---> This could be added later on, to re organise the index (Refer to pandas DropDuplicate method) +} + +// DropDuplicates is the method where users can drop duplicate values in the dataframe. +// It will return a cloned DataFrame by default and only keep the FIRST occurence of the duplicate value. +// +// Parameters: +// - options: The DropDuplicatesOption struct to optionally add parameters to this method. +// +// Returns: +// - *DataFrame: The original dataframe with no duplicates if the inplace option in DropDuplicatesOption is true. +// - *DataFrame (cloned): A cloned DataFrame if the inplace option in DropDuplicatesOption is false. +func (df *DataFrame) DropDuplicates(options ...DropDuplicatesOption) (*DataFrame, error) { + var colNames []string + finalOptions := DropDuplicatesOption{Keep: "first", Inplace: false} + newDf := NewDataFrame() + + if len(options) > 0 { + userOpt := options[0] + + // only overwrite Subset if user provided one + if len(userOpt.Subset) > 0 { + finalOptions.Subset = userOpt.Subset + } + + // only overwrite Keep if user provided one (not empty) + if userOpt.Keep != "" { + finalOptions.Keep = userOpt.Keep + } + + finalOptions.Inplace = userOpt.Inplace + } + + if len(finalOptions.Subset) > 0 { + // if the options has columns to specifically target + colNames = finalOptions.Subset + } else { + // else just do it for all columns + colNames = df.ColumnNames() + } + + // find which indexes in a column to keep (which rows to keep), prob create another helper function to do this + seen := make(map[string]bool) + indexesToKeep := []int{} + + switch finalOptions.Keep { + case "first": + // logic for finalOptions.keep = "first" + for i := 0; i < df.Nrows(); i++ { + rowKey, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if !seen[rowKey] { + seen[rowKey] = true + indexesToKeep = append(indexesToKeep, i) + } + } + + case "last": + // logic for finalOptions.keep = "last" + for i := df.Nrows() - 1; i >= 0; i-- { + rowKey, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if !seen[rowKey] { + seen[rowKey] = true + indexesToKeep = append(indexesToKeep, i) + } + } + sort.Ints(indexesToKeep) + + case "none": + // logic for finalOptions.keep = "none" + + // count all the times a specific row has occured before + counts := make(map[string]int) + for i := 0; i < df.Nrows(); i++ { + key, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + counts[key]++ + } + + // only keep the indexes where the count == 1 + for i := 0; i < df.Nrows(); i++ { + key, err := df.getRowKey(i, colNames) + if err != nil { + return nil, fmt.Errorf("Error, could not get row key: %v", err) + } + if counts[key] == 1 { + indexesToKeep = append(indexesToKeep, i) + } + } + + } + + for _, colName := range df.ColumnNames() { + // iterate through each column and get the indexes + // then directly change it in the current dataframe + // or add the new data to a new dataframe to return. + newData, err := df.getSubSlice(colName, indexesToKeep) + + if err != nil { + return nil, fmt.Errorf("Error, could not get sub slice: %v", err) + } + + // use inplace option to check whether we want to clone or modify the current dataframe + if finalOptions.Inplace { + // replace the existing data with the new data + df.Columns[colName].Data = newData + } else { + err := newDf.AddColumn(ConvertToAnyColumn(NewColumn(colName, newData))) + if err != nil { + return nil, fmt.Errorf("Error trying to add column to new DataFrame: %v", err) + } + } + } + + if finalOptions.Inplace { + return df, nil + } + + return newDf, nil + // return the DataFrame +} + +// getRowKey is a helper function to return a unique key for a row +func (df *DataFrame) getRowKey(rowIndex int, colNames []string) (string, error) { + + var builder strings.Builder + + for _, name := range colNames { + col, ok := df.Columns[name] + if !ok { + return "", fmt.Errorf("Column %s not found", name) + } + value := col.Data[rowIndex] + + // add the col name to prevent similar values but different column cases + builder.WriteString(name) + builder.WriteString(":") + + if value == nil { + builder.WriteString("nil") + } else { + builder.WriteString(fmt.Sprintf("%v", value)) + } + + builder.WriteString("|") + } + + return builder.String(), nil +} + +// getSubSlice is a method get a portion of an existing column and returns a slice +func (df *DataFrame) getSubSlice(colName string, indexesToKeep []int) ([]any, error) { + col, ok := df.Columns[colName] + if !ok { + return nil, fmt.Errorf("Column %s not found", colName) + } + + finalRows := make([]any, len(indexesToKeep)) + + for i, index := range indexesToKeep { + // use direct assignment instead of appending because appending costs more + finalRows[i] = col.Data[index] + } + + return finalRows, nil +} diff --git a/goframe.go b/goframe.go index ef284fb..a063003 100644 --- a/goframe.go +++ b/goframe.go @@ -17,6 +17,7 @@ type MultiIndex = df.MultiIndex type GroupedDataFrame = df.GroupedDataFrame type DataFrameSorter = df.DataFrameSorter type FuncType = df.FuncType +type DropDuplicatesOption = df.DropDuplicatesOption // Column is re-exported as a generic type alias type Column[T any] = df.Column[T] diff --git a/goframe_tests/dataframe_test.go b/goframe_tests/dataframe_test.go index 47e8978..f5b07b3 100644 --- a/goframe_tests/dataframe_test.go +++ b/goframe_tests/dataframe_test.go @@ -1378,6 +1378,295 @@ func TestDataFrameSortValues(t *testing.T) { }) } +// MARK: DropDuplicates +func TestDropDuplicates(t *testing.T) { + // This helper returns a fresh DF every time it is called + setupDF := func() *goframe.DataFrame { + df := goframe.NewDataFrame() + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("name", []string{"Charlie", "Alice", "Alice"}))) + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("age", []int{35, 25, 25}))) + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("score", []float64{90.5, 95.0, 95.0}))) + return df + } + t.Run("DefaultOptionsTest", func(t *testing.T) { + df := setupDF() + finalDf, err := df.DropDuplicates() + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + numberOfRows := finalDf.Nrows() + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i, val := range scoreCol.Data { + if val.(float64) != expectedScores[i] { + t.Errorf("Expected score at index %d to be %v, got %v", i, expectedScores[i], val) + } + } + + for i, val := range nameCol.Data { + if val.(string) != expectedNames[i] { + t.Errorf("Expected name at index %d to be %s, got %v", i, expectedNames[i], val) + } + } + + for i, val := range ageCol.Data { + if val.(int) != expectedAge[i] { + t.Errorf("Expected age at index %d to be %v, got %v", i, expectedAge[i], val) + } + } + }) + + t.Run("Subset Option", func(t *testing.T) { + + df2 := goframe.NewDataFrame() + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("name", []string{"Charlie", "Alice", "Bob"}))) + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("age", []int{35, 25, 25}))) // age is the same for both Alice and Bob + df2.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("score", []float64{90.5, 95.0, 80.0}))) + + dropDuplicateOption := goframe.DropDuplicatesOption{ + Subset: []string{"age"}, + } + finalDf, err := df2.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + numberOfRows := finalDf.Nrows() + + if numberOfRows != 2 { + t.Errorf("Expected 1 row but got %v", numberOfRows) + } + + if len(scoreCol.Data) != len(expectedScores) { + t.Fatalf("Column length mismatch: got %d, want %d", len(scoreCol.Data), len(expectedScores)) + } + + for i, val := range scoreCol.Data { + if val.(float64) != expectedScores[i] { + t.Errorf("Expected score at index %d to be %v, got %v", i, expectedScores[i], val) + } + } + + for i, val := range nameCol.Data { + if val.(string) != expectedNames[i] { + t.Errorf("Expected name at index %d to be %s, got %v", i, expectedNames[i], val) + } + } + + for i, val := range ageCol.Data { + if val.(int) != expectedAge[i] { + t.Errorf("Expected age at index %d to be %v, got %v", i, expectedAge[i], val) + } + } + }) + + // + t.Run("KeepOptions", func(t *testing.T) { + df := setupDF() + df.AddColumn(goframe.ConvertToAnyColumn(goframe.NewColumn("index", []int{0, 1, 2}))) + t.Run("KeepFirst", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "first", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + expectedIndex := []int{0, 1} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 2 + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + + t.Run("KeepLast", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "last", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie", "Alice"} + expectedAge := []int{35, 25} + expectedScores := []float64{90.5, 95.0} + expectedIndex := []int{0, 2} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 2 + + if numberOfRows != 2 { + t.Errorf("Expected 2 rows but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + + t.Run("KeepNone", func(t *testing.T) { + dropDuplicateOption := goframe.DropDuplicatesOption{ + Keep: "none", + Subset: []string{"name", "age", "score"}, + } + finalDf, err := df.DropDuplicates(dropDuplicateOption) + if err != nil { + t.Errorf("Error executing DropDuplicates method, %v", err) + } + + expectedNames := []string{"Charlie"} + expectedAge := []int{35} + expectedScores := []float64{90.5} + expectedIndex := []int{0} + + scoreCol, _ := finalDf.Select("score") + nameCol, _ := finalDf.Select("name") + ageCol, _ := finalDf.Select("age") + indexCol, _ := finalDf.Select("index") + numberOfRows := finalDf.Nrows() + + expectedRows := 1 + + if numberOfRows != 1 { + t.Errorf("Expected 1 row but got %v", numberOfRows) + } + + for i := range expectedRows { + // Name check + if nameCol.Data[i].(string) != expectedNames[i] { + t.Errorf("Index %d: expected name %s, got %v", i, expectedNames[i], nameCol.Data[i]) + } + // Age check + if ageCol.Data[i].(int) != expectedAge[i] { + t.Errorf("Index %d: expected age %d, got %v", i, expectedAge[i], ageCol.Data[i]) + } + // Score check + if scoreCol.Data[i].(float64) != expectedScores[i] { + t.Errorf("Index %d: expected score %v, got %v", i, expectedScores[i], scoreCol.Data[i]) + } + // Index check + if indexCol.Data[i].(int) != expectedIndex[i] { + t.Errorf("Index %d: expected original index %d, got %v", i, expectedIndex[i], indexCol.Data[i]) + } + } + }) + }) + + t.Run("InplaceTrue_ModifiesOriginal", func(t *testing.T) { + df := setupDF() + + _, err := df.DropDuplicates(goframe.DropDuplicatesOption{Inplace: true}) + if err != nil { + t.Fatalf("Operation failed: %v", err) + } + + // check 1: the original 'df' variable must now have fewer rows + expectedRows := 2 + if df.Nrows() != expectedRows { + t.Errorf("Expected original DF to be modified to %d rows, but got %d", expectedRows, df.Nrows()) + } + + // check 2: verify the data inside the original 'df' is the cleaned data + nameCol, _ := df.Select("name") + if len(nameCol.Data) != expectedRows { + t.Errorf("Original column data length mismatch: expected %d, got %d", expectedRows, len(nameCol.Data)) + } + + }) + + t.Run("InplaceFalse_PreservesOriginal", func(t *testing.T) { + df := setupDF() // Start with 3 rows + + newDf, err := df.DropDuplicates(goframe.DropDuplicatesOption{Inplace: false}) + if err != nil { + t.Fatalf("Operation failed: %v", err) + } + + // check 1: the new DataFrame should be cleaned (2 rows) + if newDf.Nrows() != 2 { + t.Errorf("New DataFrame expected 2 rows, got %d", newDf.Nrows()) + } + + // check 2: the original DataFrame MUST still have 3 rows + if df.Nrows() != 3 { + t.Errorf("Original DataFrame was accidentally modified! Expected 3 rows, got %d", df.Nrows()) + } + }) + +} + // MARK: Helper Functions /*