1use super::utils;
2#[cfg(feature = "sqlite")]
3use crate::database::QueryPlan;
4use itertools::Itertools;
5use nu_engine::command_prelude::*;
6use nu_protocol::PipelineMetadata;
7use nu_utils::IgnoreCaseExt;
8use std::collections::{HashMap, hash_map::IntoIter};
9
10#[derive(Clone)]
11pub struct Uniq;
12
13impl Command for Uniq {
14 fn name(&self) -> &str {
15 "uniq"
16 }
17
18 fn signature(&self) -> Signature {
19 Signature::build("uniq")
20 .input_output_types(vec![(
21 Type::List(Box::new(Type::Any)),
22 Type::List(Box::new(Type::Any)),
23 )])
24 .switch(
25 "count",
26 "Return a table containing the distinct input values together with their counts.",
27 Some('c'),
28 )
29 .switch(
30 "repeated",
31 "Return the input values that occur more than once.",
32 Some('d'),
33 )
34 .switch(
35 "ignore-case",
36 "Compare input values case-insensitively.",
37 Some('i'),
38 )
39 .switch(
40 "unique",
41 "Return the input values that occur once only.",
42 Some('u'),
43 )
44 .category(Category::Filters)
45 }
46
47 fn description(&self) -> &str {
48 "Return the distinct values in the input."
49 }
50
51 fn search_terms(&self) -> Vec<&str> {
52 vec!["distinct", "deduplicate", "count"]
53 }
54
55 fn run(
56 &self,
57 engine_state: &EngineState,
58 stack: &mut Stack,
59 call: &Call,
60 mut input: PipelineData,
61 ) -> Result<PipelineData, ShellError> {
62 let head = call.head;
63
64 #[cfg(feature = "sqlite")]
65 if !call.has_flag(engine_state, stack, "count")?
67 && !call.has_flag(engine_state, stack, "repeated")?
68 && !call.has_flag(engine_state, stack, "unique")?
69 && !call.has_flag(engine_state, stack, "ignore-case")?
70 && let PipelineData::Value(Value::Custom { val, .. }, metadata) = &input
71 && let Some(plan) = QueryPlan::try_from_any(val.as_any())
72 {
73 let plan = plan.with_distinct();
74 return plan
75 .execute(call.head)
76 .map(|data| data.set_metadata(metadata.clone()));
77 }
78
79 let mapper = Box::new(
80 move |ms: ItemMapperState| -> Result<ValueCounter, ShellError> {
81 Ok(item_mapper(ms.item, ms.flag_ignore_case, ms.index, head))
82 },
83 );
84
85 let metadata = input.take_metadata();
86 uniq(
87 engine_state,
88 stack,
89 call,
90 input.into_iter().collect(),
91 mapper,
92 metadata,
93 )
94 }
95
96 fn examples(&self) -> Vec<Example<'_>> {
97 vec![
98 Example {
99 description: "Return the distinct values of a list/table (remove duplicates so that each value occurs once only).",
100 example: "[2 3 3 4] | uniq",
101 result: Some(Value::list(
102 vec![Value::test_int(2), Value::test_int(3), Value::test_int(4)],
103 Span::test_data(),
104 )),
105 },
106 Example {
107 description: "Return the input values that occur more than once.",
108 example: "[1 2 2] | uniq -d",
109 result: Some(Value::list(vec![Value::test_int(2)], Span::test_data())),
110 },
111 Example {
112 description: "Return the input values that occur once only.",
113 example: "[1 2 2] | uniq --unique",
114 result: Some(Value::list(vec![Value::test_int(1)], Span::test_data())),
115 },
116 Example {
117 description: "Ignore differences in case when comparing input values.",
118 example: "['hello' 'goodbye' 'Hello'] | uniq --ignore-case",
119 result: Some(Value::test_list(vec![
120 Value::test_string("hello"),
121 Value::test_string("goodbye"),
122 ])),
123 },
124 Example {
125 description: "Return a table containing the distinct input values together with their counts.",
126 example: "[1 2 2] | uniq --count",
127 result: Some(Value::test_list(vec![
128 Value::test_record(record! {
129 "value" => Value::test_int(1),
130 "count" => Value::test_int(1),
131 }),
132 Value::test_record(record! {
133 "value" => Value::test_int(2),
134 "count" => Value::test_int(2),
135 }),
136 ])),
137 },
138 ]
139 }
140}
141
142pub struct ItemMapperState {
143 pub item: Value,
144 pub flag_ignore_case: bool,
145 pub index: usize,
146 pub head: Span,
147}
148
149fn item_mapper(item: Value, flag_ignore_case: bool, index: usize, head: Span) -> ValueCounter {
150 ValueCounter::new(item, flag_ignore_case, index, head)
151}
152
153pub struct ValueCounter {
154 val: Value,
155 val_to_compare: Value,
156 count: i64,
157 index: usize,
158}
159
160impl PartialEq<Self> for ValueCounter {
161 fn eq(&self, other: &Self) -> bool {
162 self.val == other.val
163 }
164}
165
166impl ValueCounter {
167 fn new(val: Value, flag_ignore_case: bool, index: usize, head: Span) -> Self {
168 Self::new_vals_to_compare(val.clone(), flag_ignore_case, val, index, head)
169 }
170 pub fn new_vals_to_compare(
171 val: Value,
172 flag_ignore_case: bool,
173 vals_to_compare: Value,
174 index: usize,
175 head: Span,
176 ) -> Self {
177 ValueCounter {
178 val,
179 val_to_compare: if flag_ignore_case {
180 clone_to_folded_case(&vals_to_compare.with_span(head))
181 } else {
182 vals_to_compare.with_span(head)
183 },
184 count: 1,
185 index,
186 }
187 }
188}
189
190fn clone_to_folded_case(value: &Value) -> Value {
191 let span = value.span();
192 match value {
193 Value::String { val: s, .. } => Value::string(s.clone().to_folded_case(), span),
194 Value::List { vals: vec, .. } => {
195 Value::list(vec.iter().map(clone_to_folded_case).collect(), span)
196 }
197 Value::Record { val: record, .. } => Value::record(
198 record
199 .iter()
200 .map(|(k, v)| (k.to_owned(), clone_to_folded_case(v)))
201 .collect(),
202 span,
203 ),
204 other => other.clone(),
205 }
206}
207
208fn generate_results_with_count(head: Span, uniq_values: Vec<ValueCounter>) -> Vec<Value> {
209 uniq_values
210 .into_iter()
211 .map(|item| {
212 Value::record(
213 record! {
214 "value" => item.val,
215 "count" => Value::int(item.count, head),
216 },
217 head,
218 )
219 })
220 .collect()
221}
222
223pub fn uniq(
224 engine_state: &EngineState,
225 stack: &mut Stack,
226 call: &Call,
227 input: Vec<Value>,
228 item_mapper: Box<dyn Fn(ItemMapperState) -> Result<ValueCounter, ShellError>>,
229 metadata: Option<PipelineMetadata>,
230) -> Result<PipelineData, ShellError> {
231 let head = call.head;
232 let flag_show_count = call.has_flag(engine_state, stack, "count")?;
233 let flag_show_repeated = call.has_flag(engine_state, stack, "repeated")?;
234 let flag_ignore_case = call.has_flag(engine_state, stack, "ignore-case")?;
235 let flag_only_uniques = call.has_flag(engine_state, stack, "unique")?;
236
237 let flag_keep_last = call.has_flag(engine_state, stack, "keep-last")?;
239
240 let signals = engine_state.signals().clone();
241 let mut uniq_values = input
242 .into_iter()
243 .enumerate()
244 .map_while(|(index, item)| {
245 if signals.interrupted() {
246 return None;
247 }
248 Some(item_mapper(ItemMapperState {
249 item,
250 flag_ignore_case,
251 index,
252 head,
253 }))
254 })
255 .try_fold(
256 HashMap::<String, ValueCounter>::new(),
257 |mut counter, item| -> Result<_, ShellError> {
258 let item = item?;
259 let key = utils::value_to_key(engine_state, &item.val_to_compare, head)?;
260
261 match counter.get_mut(&key) {
262 Some(x) => {
263 if flag_keep_last {
264 x.val = item.val;
265 }
266 x.count += 1;
267 }
268 None => {
269 counter.insert(key, item);
270 }
271 };
272 Ok(counter)
273 },
274 )?;
275
276 if flag_show_repeated {
277 uniq_values.retain(|_v, value_count_pair| value_count_pair.count > 1);
278 }
279
280 if flag_only_uniques {
281 uniq_values.retain(|_v, value_count_pair| value_count_pair.count == 1);
282 }
283
284 let uniq_values = sort(uniq_values.into_iter());
285
286 let result = if flag_show_count {
287 generate_results_with_count(head, uniq_values)
288 } else {
289 uniq_values.into_iter().map(|v| v.val).collect()
290 };
291
292 Ok(Value::list(result, head).into_pipeline_data_with_metadata(metadata))
293}
294
295fn sort(iter: IntoIter<String, ValueCounter>) -> Vec<ValueCounter> {
296 iter.map(|item| item.1)
297 .sorted_by(|a, b| a.index.cmp(&b.index))
298 .collect()
299}
300
301#[cfg(test)]
302mod test {
303 use super::*;
304
305 #[test]
306 fn test_examples() -> nu_test_support::Result {
307 nu_test_support::test().examples(Uniq)
308 }
309}