weld 0.4.0

Weld is a language and runtime for improving the performance of data-intensive applications.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

//! Tests for `WeldContext`.
//!
//! These tests reuse builders across Weld programs by reusing contexts.

use weld;

use weld::data::*;
use weld::*;

mod common;
use crate::common::*;

#[test]
fn reuse_appender_test() {
    let ref conf = default_conf();
    let ref mut context = WeldContext::new(conf).unwrap();

    let program1 = "|v: vec[i32]| for(v, appender[i32], |b,i,e| merge(b,e))";

    let input_vec = vec![1, 2, 3, 4, 5];
    let input_data = WeldVec::from(&input_vec);

    let ref input_value = WeldValue::new_from_data(&input_data as *const _ as Data);

    let module = WeldModule::compile(program1, conf).unwrap();
    let ret_value = unsafe { module.run(context, input_value).unwrap() };
    let data = ret_value.data() as *const Appender<i32>;
    let appender = unsafe { (*data).clone() };

    // The input for the second program.
    #[derive(Clone)]
    #[allow(dead_code)]
    #[repr(C)]
    struct Program2Args {
        vec: WeldVec<i32>,
        appender: Appender<i32>,
    }

    let program2 = "|v: vec[i32], a: appender[i32]|
        result(for(v, a, |b,i,e| merge(b,e)))";

    let ref input_data = Program2Args {
        vec: input_data,
        appender: appender,
    };

    let ref input_value = WeldValue::new_from_data(input_data as *const _ as Data);
    let module = WeldModule::compile(program2, conf).unwrap();

    // Run new module with same context.
    let ret_value = unsafe { module.run(context, input_value).unwrap() };
    let data = ret_value.data() as *const WeldVec<i32>;

    let result = unsafe { (*data).clone() };

    // Check correctness
    assert_eq!(result.len as usize, input_vec.len() * 2);
    for i in 0..result.len as isize {
        assert_eq!(
            unsafe { *result.data.offset(i) as i32 },
            input_vec[(i % input_vec.len() as isize) as usize]
        );
    }
}

#[test]
fn reuse_groupmerger_test() {
    #[derive(Clone)]
    #[allow(dead_code)]
    #[repr(C)]
    struct Program1Args {
        keys: WeldVec<i32>,
        vals: WeldVec<i32>,
    }

    let ref conf = default_conf();
    let ref mut context = WeldContext::new(conf).unwrap();

    let program1 = "|keys: vec[i32], vals: vec[i32]|
        for(zip(keys, vals), groupmerger[i32,i32], |b,i,e| merge(b,e))";

    let keys = vec![1, 2, 2, 3, 3, 1];
    let vals = vec![2, 3, 4, 1, 0, 2];
    let input_data = Program1Args {
        keys: WeldVec::from(&keys),
        vals: WeldVec::from(&vals),
    };

    let ref input_value = WeldValue::new_from_data(&input_data as *const _ as Data);

    let module = WeldModule::compile(program1, conf).unwrap();
    let ret_value = unsafe { module.run(context, input_value).unwrap() };
    let data = ret_value.data() as *const GroupMerger<i32, i32>;
    let groupmerger = unsafe { (*data).clone() };

    // The input for the second program.
    #[derive(Clone)]
    #[allow(dead_code)]
    #[repr(C)]
    struct Program2Args {
        keys: WeldVec<i32>,
        vals: WeldVec<i32>,
        builder: GroupMerger<i32, i32>,
    }

    let program2 = "|keys: vec[i32], vals: vec[i32], gm: groupmerger[i32,i32]|
        tovec(result(for(zip(keys,vals), gm, |b,i,e| merge(b,e))))";

    let ref input_data = Program2Args {
        keys: WeldVec::from(&keys),
        vals: WeldVec::from(&vals),
        builder: groupmerger,
    };

    let ref input_value = WeldValue::new_from_data(input_data as *const _ as Data);
    let module = WeldModule::compile(program2, conf).unwrap();

    // Run new module with same context.
    let ret_value = unsafe { module.run(context, input_value).unwrap() };
    let data = ret_value.data() as *const WeldVec<Pair<i32, WeldVec<i32>>>;

    let result = unsafe { (*data).clone() };

    let expect: Vec<(i32, Vec<i32>)> = vec![
        (1, vec![2, 2, 2, 2]),
        (2, vec![3, 4, 3, 4]),
        (3, vec![1, 0, 1, 0]),
    ];

    // Check correctness
    assert_eq!(result.len as usize, expect.len());

    let mut res: Vec<(i32, Vec<i32>)> = (0..result.len)
        .into_iter()
        .map(|x| {
            let key = unsafe { (*result.data.offset(x as isize)).ele1 };
            let val = unsafe { ((*result.data.offset(x as isize)).ele2).clone() };
            let vec: Vec<i32> = (0..val.len)
                .into_iter()
                .map(|y| unsafe { *val.data.offset(y as isize) })
                .collect();
            (key, vec)
        })
        .collect();
    res.sort_by_key(|a| a.0);
    assert_eq!(res, expect);
}