1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
//!
//! # Network Service Handler
//!
//! Operations to deal with the requests from client.
//!
//! 与 Server 同名接口的对外表现完全一致.
//!

mod add_env;
pub(crate) mod sync;

use crate::{def::*, send_err, send_ok, CFG, PROXY, SOCK_MID};
use add_env::add_env;
use async_std::{net::SocketAddr, task};
use lazy_static::lazy_static;
use myutil::{err::*, *};
use parking_lot::RwLock;
use serde::Deserialize;
use serde::Serialize;
use std::{
    collections::HashMap,
    mem,
    sync::{
        atomic::{AtomicU64, Ordering},
        Arc,
    },
};
use ttserver_def::*;

lazy_static! {
    static ref ENV_MAP: Arc<RwLock<HashMap<EnvId, Vec<SocketAddr>>>> =
        Arc::new(RwLock::new(map! {}));
    static ref SLAVE_INFO: Arc<RwLock<HashMap<SocketAddr, RespGetServerInfo>>> =
        Arc::new(RwLock::new(map! {}));
}

type Ops = fn(usize, SocketAddr, Vec<u8>) -> Result<()>;
include!("included_ops_map.rs");

/// 将客户端的请求,
/// 分发至后台的各个 Slave Server.
#[macro_export(crate)]
macro_rules! fwd_to_slave {
    (@@@$ops_id: expr, $req: expr, $peeraddr: expr, $cb: tt, $addr_set: expr) => {{
        let num_to_wait = $addr_set.len();
        let proxy_uuid = gen_proxy_uuid();

        register_resp_hdr(num_to_wait, $cb, $peeraddr, $req.uuid, proxy_uuid);

        let cli_id = $req.cli_id.take().unwrap_or_else(||$peeraddr.ip().to_string());
        send_req_to_slave($ops_id, Req::newx(proxy_uuid, Some(cli_id), mem::take(&mut $req.msg)), $addr_set)
            .c(d!())
    }};
    (@@$ops_id: expr, $req: expr, $peeraddr: expr, $cb: tt, $addr_set: expr) => {{
        let mut req = $req;
        fwd_to_slave!(@@@$ops_id, req, $peeraddr, $cb, $addr_set)
            .or_else(|e| send_err!(req.uuid, e, $peeraddr))
    }};
    (@$ops_id: expr, $orig_req: expr, $req_kind: ty, $peeraddr: expr, $cb: tt) => {{
        let req = serde_json::from_slice::<$req_kind>(&$orig_req).c(d!())?;
        let addr_set = if let Some(set) = ENV_MAP.read().get(&req.msg.env_id) {
            set.clone()
        } else {
            return send_err!(req.uuid, eg!("ENV not exists!"), $peeraddr);
        };
        fwd_to_slave!(@@$ops_id, req, $peeraddr, $cb, &addr_set)
    }};
    ($ops_id: expr, $orig_req: expr, $req_kind: ty, $peeraddr: expr) => {{
        fwd_to_slave!(@$ops_id, $orig_req, $req_kind, $peeraddr, resp_cb_simple)
    }};
}

/// 注册 Cli, 一般无需调用,
/// 创建 Env 时若发现 Cli 不存在会自动创建之,
/// 此接口在 Proxy 中实现为"什么都不做, 直接返回成功".
fn register_client_id(
    _ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    let req = serde_json::from_slice::<Req<&str>>(&request).c(d!())?;
    let resp = Resp {
        uuid: req.uuid,
        status: RetStatus::Success,
        msg: vct![],
    };
    send_ok!(req.uuid, resp, peeraddr)
}

/// 获取服务端的资源分配信息,
/// 直接从定时任务的结果中提取, 不做实时请求.
fn get_server_info(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    // 汇聚各 Slave 的信息
    fn cb(r: &mut SlaveRes) {
        info_omit!(resp_cb_merge::<RespGetServerInfo>(r));
    }

    let req = serde_json::from_slice::<Req<&str>>(&request).c(d!())?;
    let addr_set = CFG.server_addr_set.clone();

    fwd_to_slave!(@@ops_id, req, peeraddr, cb, &addr_set)
}

/// 获取服务端已存在的 Env 概略信息
fn get_env_list(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    // 汇聚各 Slave 的信息
    fn cb(r: &mut SlaveRes) {
        info_omit!(resp_cb_merge::<RespGetEnvList>(r));
    }

    let req = serde_json::from_slice::<Req<&str>>(&request).c(d!())?;
    let addr_set = CFG.server_addr_set.clone();

    fwd_to_slave!(@@ops_id, req, peeraddr, cb, &addr_set)
}

// 获取服务端已存在的 Env 详细信息
fn get_env_info(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqGetEnvInfo,
    }

    // 汇聚各 Slave 的信息
    fn cb(r: &mut SlaveRes) {
        info_omit!(resp_cb_merge::<RespGetEnvInfo>(r));
    }

    let req = serde_json::from_slice::<MyReq>(&request).c(d!())?;
    let addr_set = {
        let lk = ENV_MAP.read();
        let res = req
            .msg
            .env_set
            .iter()
            .filter_map(|env_id| lk.get(env_id))
            .flatten()
            .copied()
            .collect::<Vec<_>>();
        if res.is_empty() {
            let msg: HashMap<String, RespGetEnvInfo> = map! {};
            return send_ok!(req.uuid, msg, peeraddr);
        } else {
            res
        }
    };

    fwd_to_slave!(@@ops_id, req, peeraddr, cb, &addr_set)
}

/// 从已有 ENV 中踢出指定的 VM 实例
fn update_env_kick_vm(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqUpdateEnvKickVm,
    }

    fwd_to_slave!(ops_id, request, MyReq, peeraddr)
}

/// 更新已有 Env 的生命周期
fn update_env_lifetime(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqUpdateEnvLife,
    }

    let req = serde_json::from_slice::<MyReq>(&request).c(d!())?;

    if let Some(set) = ENV_MAP.read().get(&req.msg.env_id) {
        fwd_to_slave!(@@ops_id, req, peeraddr, resp_cb_simple, set)
    } else {
        send_err!(req.uuid, eg!("ENV not exists!"), peeraddr)
    }
}

/// 删除 Env
fn del_env(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqDelEnv,
    }

    let req = serde_json::from_slice::<MyReq>(&request).c(d!())?;
    let addr_set = if let Some(set) = ENV_MAP.write().remove(&req.msg.env_id) {
        set
    } else {
        return send_ok!(req.uuid, "Success!", peeraddr);
    };

    fwd_to_slave!(@@ops_id, req, peeraddr, resp_cb_simple, &addr_set)
}

/// 获取服务端已存在的 Env 概略信息(全局)
#[inline(always)]
fn get_env_list_all(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    get_env_list(ops_id, peeraddr, request).c(d!())
}

/// 暂停运行, 让出资源
/// - 保留临时镜像和端口影射
/// - 停止所有 VM 进程
/// - 资源计数递减
fn stop_env(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqStopEnv,
    }

    fwd_to_slave!(ops_id, request, MyReq, peeraddr)
}

/// 恢复运行先前被 stop 的 ENV
/// - 启动所有 VM 进程
/// - 资源计数递增
fn start_env(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqStartEnv,
    }

    fwd_to_slave!(ops_id, request, MyReq, peeraddr)
}

/// 更新已有 ENV 中资源配置信息
fn update_env_resource(
    ops_id: usize,
    peeraddr: SocketAddr,
    request: Vec<u8>,
) -> Result<()> {
    #[derive(Deserialize)]
    struct MyReq {
        uuid: u64,
        cli_id: Option<CliId>,
        msg: ReqUpdateEnvResource,
    }

    fwd_to_slave!(ops_id, request, MyReq, peeraddr)
}

/// 生成不重复的 uuid
fn gen_proxy_uuid() -> u64 {
    lazy_static! {
        static ref UUID: AtomicU64 = AtomicU64::new(9999);
    }
    UUID.fetch_add(1, Ordering::Relaxed)
}

// 用于回复 Client 的通用回调
fn resp_cb_simple(r: &mut SlaveRes) {
    if 0 < r.num_to_wait {
        send_err!(
            r.uuid,
            eg!("Not all slave-server[s] reponsed!"),
            r.peeraddr
        )
        .unwrap_or_else(|e| p(e));
    } else if r.msg.values().any(|v| v.status == RetStatus::Fail) {
        let msg = r
            .msg
            .values()
            .filter(|v| v.status == RetStatus::Fail)
            .map(|v| v.to_string())
            .collect::<Vec<_>>()
            .join(" ;; ");
        send_err!(r.uuid, eg!(msg), r.peeraddr).unwrap_or_else(|e| p(e));
    } else {
        send_ok!(r.uuid, "Success!", r.peeraddr).unwrap_or_else(|e| p(e));
    }
}

// 汇聚各 Slave 的信息, 回复给 Client;
// 该回调仅用于查询类接口, 采用 “尽力而为” 模式, 部分返回即视为成功.
fn resp_cb_merge<'a, T: Serialize + Deserialize<'a>>(
    r: &'a mut SlaveRes,
) -> Result<()> {
    if 0 < r.num_to_wait {
        p(eg!("Not all slave-server[s] reponsed!"));
    }

    // if r.msg.values().any(|v| v.status == RetStatus::Fail) {
    //     p(eg!("Some slave-server[s] got error!"));
    // }

    let res = r
        .msg
        .iter()
        .filter(|(_, raw)| raw.status == RetStatus::Success)
        .filter_map(|(slave, raw)| {
            info!(serde_json::from_slice::<HashMap<ServerAddr, T>>(&raw.msg))
                .ok()
                .and_then(|resp| resp.into_iter().next())
                .map(|resp| (slave.to_string(), resp.1))
        })
        .collect::<HashMap<_, _>>();

    send_ok!(r.uuid, res, r.peeraddr)
}

/// 分发请求至各 Slave Server
fn send_req_to_slave<T: Serialize>(
    ops_id: usize,
    req: Req<T>,
    slave_set: &[SocketAddr],
) -> Result<()> {
    let mut req_bytes = serde_json::to_vec(&req).c(d!())?;
    let mut body =
        format!("{id:>0width$}", id = ops_id, width = OPS_ID_LEN).into_bytes();
    body.append(&mut req_bytes);

    macro_rules! do_send {
        ($body: expr, $slave: expr) => {
            task::spawn(async move {
                info_omit!(SOCK_MID.send_to(&$body, $slave).await);
            });
        };
    }

    if 1 < slave_set.len() {
        for slave in slave_set.iter().skip(1).copied() {
            let b = body.clone();
            do_send!(b, slave);
        }
    } else if slave_set.is_empty() {
        return Ok(());
    }

    // 单独处理第一个 slave, 避免无谓的 clone
    let first_slave = slave_set[0];
    do_send!(body, first_slave);

    Ok(())
}

/// 将待回复的 handler 注册到 Proxy 中
fn register_resp_hdr(
    num_to_wait: usize,
    cb: fn(&mut SlaveRes),
    peeraddr: SocketAddr,
    orig_uuid: UUID,
    proxy_uuid: UUID,
) {
    let ts = ts!();
    let idx = ts as usize % TIMEOUT_SECS;
    let sr = SlaveRes {
        msg: map! {},
        num_to_wait,
        start_ts: ts,
        do_resp: cb,
        peeraddr,
        uuid: orig_uuid,
    };
    let mut proxy = PROXY.lock();
    proxy.idx_map.insert(proxy_uuid, idx);

    // 清理已失效的 Bucket 内容,
    // 不能单纯依靠"清理线程"去处理,
    // 会存在错误地更新了本该过期的 Bucket 时间戳的现象,
    // 导致"清理线程"无法正确识别将被清理的对象.
    if ts != proxy.buckets[idx].ts {
        mem::take(&mut proxy.buckets[idx]).res.keys().for_each(|k| {
            proxy.idx_map.remove(k);
        });
    }

    // 而后才能更新时间戳
    proxy.buckets[idx].ts = ts;

    proxy.buckets[idx].res.insert(proxy_uuid, sr);
}