abd-clam 0.11.0

Clustered Learning of Approximate Manifolds
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

import math
import pathlib

from abd_clam import anomaly_detection
from abd_clam import metric
from abd_clam.anomaly_detection import graph_scorers
from abd_clam.core import cluster_criteria

from . import anomaly_data


def default_training(data_dir: pathlib.Path, output_dir: pathlib.Path):
    raw_datasets = [
        anomaly_data.AnomalyData.load(data_dir, name)
        for name in anomaly_data.TRAINING_SET
    ]
    datasets = [
        anomaly_detection.anomaly_dataset.AnomalyTabular(
            data=data.normalized_features,
            scores=data.scores,
            name=name,
        )
        for name, data in zip(anomaly_data.TRAINING_SET, raw_datasets)
    ]

    metrics = [
        metric.ScipyMetric("euclidean"),
        metric.ScipyMetric("cityblock"),
    ]

    spaces = [
        anomaly_detection.anomaly_space.AnomalyTabularSpace(d, m, use_cache=False)
        for d in datasets
        for m in metrics
    ]
    spaces_criteria = [
        (s, [cluster_criteria.MinPoints(1 + int(math.log2(s.data.cardinality)))])
        for s in spaces
    ]

    models_kwargs = [
        (anomaly_detection.meta_ml.MetaDT, {}),
        (anomaly_detection.meta_ml.MetaLR, {}),
    ]

    scorers = [
        graph_scorers.ClusterCardinality(),
        graph_scorers.ComponentCardinality(),
        graph_scorers.VertexDegree(),
        graph_scorers.ParentCardinality(depth_weight=lambda d: 1 / (d**0.5)),
        graph_scorers.GraphNeighborhood(eccentricity_fraction=0.25),
        graph_scorers.StationaryProbabilities(steps=16),
    ]

    final_path = anomaly_detection.training.train_meta_ml(
        spaces_criteria=spaces_criteria,
        models_kwargs=models_kwargs,
        scorers=scorers,
        out_dir=output_dir,
        num_epochs=10,
        save_frequency=1,
        only_train_fast_scorers=True,
    )

    return final_path