← Back to Documentations

Signature Description Parameters 
template<typename T, typename ... Ts>
void
remove_data_by_isof(const char *col_name,
                    long num_trees = 100,
                    long max_depth = 10,
                    double threshold = 0.6,
                    normalization_type ntype =
                        normalization_type::none);
This uses Isolation Forest to detect and remove outliers in the named column and all rows corresponding to those outliers in the DataFrame.

NOTE: Based on the test below, this algorithm doesn't work very precisely -- at least not for the data sets that is being tested for. 		T: Type of the named column
Ts: The list of types for all columns. A type should be specified only once
col_name: Name of the data column
num_trees: Number of isolation trees. The original paper uses 100; 50–200 is typical. More trees -> lower variance of scores.
max_depth: Maximum depth of each tree. The original paper recommends ceil(log2(φ)) where φ is the subsample size, but passing the full column size works well in practice. Default: 10 (same as IsoTree's default).
threshold: Points with score > threshold are flagged as anomalies. The canonical threshold from the paper is 0.6 (points scoring above 0.6 are considered anomalous with moderate confidence; > 0.7 is stronger). Default is 0.6.
ntype: Normalization type to be applied as the first step
 
static void test_remove_data_by_isof()  {

    std::cout << "\nTesting remove_data_by_isof( ) ..." << std::endl;

    using MyStdDataFrame = StdDataFrame<unsigned long>;
    using StrDataFrame = StdDataFrame<std::string>;

    constexpr std::size_t   item_cnt = 1024;
    MyStdDataFrame          df;

    df.load_index(MyStdDataFrame::gen_sequence_index(0, item_cnt, 1));

    std::vector<double>   sine_col;

    sine_col.reserve(item_cnt);

    // 35 items will be affected
    //
    for (std::size_t i = 0; i < item_cnt; ++i)  {
        sine_col.push_back(std::sin(2.0 * M_PI * i / 20.0)); // Base sine wave
        if (i % 30 == 0)  sine_col.back() += 2.0;  // Inject anomalies
    }
    df.load_column("sine col", std::move(sine_col));

    MyStdDataFrame  df2 = df;

    auto    lbd = [](const unsigned long &, const double &) -> bool { return (true); };
    auto    view = df2.get_view_by_sel<double, decltype(lbd), double>("sine col", lbd);

    assert((df.get_column<double>("sine col").size() == 1024));
    assert((view.get_column<double>("sine col").size() == 1024));

    df.remove_data_by_isof<double, double>("sine col", 1000, 50, 0.6);
    assert((df.get_column<double>("sine col").size() == (1024 - 153)));

    view.remove_data_by_isof<double, double>("sine col", 1000, 50, 0.6);
    assert((view.get_column<double>("sine col").size() == (1024 - 153)));

    // Now do the same thing for IBM market data
    //
    StrDataFrame    ibm;

    try  {
        ibm.read("IBM.csv", io_format::csv2);
    }
    catch (const DataFrameError &ex)  {
        std::cout << ex.what() << std::endl;
        ::exit(-1);
    }
    ibm.get_column<double>("IBM_Close")[502] = 800.0;
    ibm.get_column<double>("IBM_Close")[1001] = 900.0;
    ibm.get_column<double>("IBM_Close")[2002] = 850.0;

    StrDataFrame    ibm2 = ibm;
    auto            ibm_lbd = [](const std::string &, const double &) -> bool { return (true); };
    auto            ibm_view = ibm2.get_view_by_sel<double, decltype(ibm_lbd), double, long>("IBM_Open", ibm_lbd);

    ibm_view.get_column<double>("IBM_Close")[502] = 800.0;
    ibm_view.get_column<double>("IBM_Close")[1001] = 900.0;
    ibm_view.get_column<double>("IBM_Close")[2002] = 850.0;

    assert((ibm.get_column<double>("IBM_Open").size() == 5031));
    assert((ibm_view.get_column<double>("IBM_Open").size() == 5031));

    ibm.remove_data_by_isof<double, double, long>("IBM_Close", 1000, 50, 0.6);
    assert((ibm.get_column<double>("IBM_Open").size() == (5031 - 40)));

    ibm_view.remove_data_by_isof<double, double, long>(
                 "IBM_Close", 1000, 50, 0.6);
    assert((ibm_view.get_column<double>("IBM_Open").size() == (5031 - 40)));
}

C++ DataFrame