1use itertools::Itertools;
2use nu_engine::command_prelude::*;
3use nu_protocol::PipelineMetadata;
4use nu_utils::IgnoreCaseExt;
5use std::collections::{HashMap, hash_map::IntoIter};
6
7#[derive(Clone)]
8pub struct Uniq;
9
10impl Command for Uniq {
11 fn name(&self) -> &str {
12 "uniq"
13 }
14
15 fn signature(&self) -> Signature {
16 Signature::build("uniq")
17 .input_output_types(vec![(
18 Type::List(Box::new(Type::Any)),
19 Type::List(Box::new(Type::Any)),
20 )])
21 .switch(
22 "count",
23 "Return a table containing the distinct input values together with their counts",
24 Some('c'),
25 )
26 .switch(
27 "repeated",
28 "Return the input values that occur more than once",
29 Some('d'),
30 )
31 .switch(
32 "ignore-case",
33 "Compare input values case-insensitively",
34 Some('i'),
35 )
36 .switch(
37 "unique",
38 "Return the input values that occur once only",
39 Some('u'),
40 )
41 .category(Category::Filters)
42 }
43
44 fn description(&self) -> &str {
45 "Return the distinct values in the input."
46 }
47
48 fn search_terms(&self) -> Vec<&str> {
49 vec!["distinct", "deduplicate", "count"]
50 }
51
52 fn run(
53 &self,
54 engine_state: &EngineState,
55 stack: &mut Stack,
56 call: &Call,
57 input: PipelineData,
58 ) -> Result<PipelineData, ShellError> {
59 let mapper = Box::new(move |ms: ItemMapperState| -> ValueCounter {
60 item_mapper(ms.item, ms.flag_ignore_case, ms.index)
61 });
62
63 let metadata = input.metadata();
64 uniq(
65 engine_state,
66 stack,
67 call,
68 input.into_iter().collect(),
69 mapper,
70 metadata,
71 )
72 }
73
74 fn examples(&self) -> Vec<Example<'_>> {
75 vec![
76 Example {
77 description: "Return the distinct values of a list/table (remove duplicates so that each value occurs once only)",
78 example: "[2 3 3 4] | uniq",
79 result: Some(Value::list(
80 vec![Value::test_int(2), Value::test_int(3), Value::test_int(4)],
81 Span::test_data(),
82 )),
83 },
84 Example {
85 description: "Return the input values that occur more than once",
86 example: "[1 2 2] | uniq -d",
87 result: Some(Value::list(vec![Value::test_int(2)], Span::test_data())),
88 },
89 Example {
90 description: "Return the input values that occur once only",
91 example: "[1 2 2] | uniq --unique",
92 result: Some(Value::list(vec![Value::test_int(1)], Span::test_data())),
93 },
94 Example {
95 description: "Ignore differences in case when comparing input values",
96 example: "['hello' 'goodbye' 'Hello'] | uniq --ignore-case",
97 result: Some(Value::test_list(vec![
98 Value::test_string("hello"),
99 Value::test_string("goodbye"),
100 ])),
101 },
102 Example {
103 description: "Return a table containing the distinct input values together with their counts",
104 example: "[1 2 2] | uniq --count",
105 result: Some(Value::test_list(vec![
106 Value::test_record(record! {
107 "value" => Value::test_int(1),
108 "count" => Value::test_int(1),
109 }),
110 Value::test_record(record! {
111 "value" => Value::test_int(2),
112 "count" => Value::test_int(2),
113 }),
114 ])),
115 },
116 ]
117 }
118}
119
120pub struct ItemMapperState {
121 pub item: Value,
122 pub flag_ignore_case: bool,
123 pub index: usize,
124}
125
126fn item_mapper(item: Value, flag_ignore_case: bool, index: usize) -> ValueCounter {
127 ValueCounter::new(item, flag_ignore_case, index)
128}
129
130pub struct ValueCounter {
131 val: Value,
132 val_to_compare: Value,
133 count: i64,
134 index: usize,
135}
136
137impl PartialEq<Self> for ValueCounter {
138 fn eq(&self, other: &Self) -> bool {
139 self.val == other.val
140 }
141}
142
143impl ValueCounter {
144 fn new(val: Value, flag_ignore_case: bool, index: usize) -> Self {
145 Self::new_vals_to_compare(val.clone(), flag_ignore_case, val, index)
146 }
147 pub fn new_vals_to_compare(
148 val: Value,
149 flag_ignore_case: bool,
150 vals_to_compare: Value,
151 index: usize,
152 ) -> Self {
153 ValueCounter {
154 val,
155 val_to_compare: if flag_ignore_case {
156 clone_to_folded_case(&vals_to_compare.with_span(Span::unknown()))
157 } else {
158 vals_to_compare.with_span(Span::unknown())
159 },
160 count: 1,
161 index,
162 }
163 }
164}
165
166fn clone_to_folded_case(value: &Value) -> Value {
167 let span = value.span();
168 match value {
169 Value::String { val: s, .. } => Value::string(s.clone().to_folded_case(), span),
170 Value::List { vals: vec, .. } => {
171 Value::list(vec.iter().map(clone_to_folded_case).collect(), span)
172 }
173 Value::Record { val: record, .. } => Value::record(
174 record
175 .iter()
176 .map(|(k, v)| (k.to_owned(), clone_to_folded_case(v)))
177 .collect(),
178 span,
179 ),
180 other => other.clone(),
181 }
182}
183
184fn sort_attributes(val: Value) -> Value {
185 let span = val.span();
186 match val {
187 Value::Record { val, .. } => {
188 let sorted = val
190 .into_owned()
191 .into_iter()
192 .sorted_by(|a, b| a.0.cmp(&b.0))
193 .collect_vec();
194
195 let record = sorted
196 .into_iter()
197 .map(|(k, v)| (k, sort_attributes(v)))
198 .collect();
199
200 Value::record(record, span)
201 }
202 Value::List { vals, .. } => {
203 Value::list(vals.into_iter().map(sort_attributes).collect_vec(), span)
204 }
205 other => other,
206 }
207}
208
209fn generate_key(engine_state: &EngineState, item: &ValueCounter) -> Result<String, ShellError> {
210 let value = sort_attributes(item.val_to_compare.clone()); nuon::to_nuon(
212 engine_state,
213 &value,
214 nuon::ToNuonConfig::default().span(Some(Span::unknown())),
215 )
216}
217
218fn generate_results_with_count(head: Span, uniq_values: Vec<ValueCounter>) -> Vec<Value> {
219 uniq_values
220 .into_iter()
221 .map(|item| {
222 Value::record(
223 record! {
224 "value" => item.val,
225 "count" => Value::int(item.count, head),
226 },
227 head,
228 )
229 })
230 .collect()
231}
232
233pub fn uniq(
234 engine_state: &EngineState,
235 stack: &mut Stack,
236 call: &Call,
237 input: Vec<Value>,
238 item_mapper: Box<dyn Fn(ItemMapperState) -> ValueCounter>,
239 metadata: Option<PipelineMetadata>,
240) -> Result<PipelineData, ShellError> {
241 let head = call.head;
242 let flag_show_count = call.has_flag(engine_state, stack, "count")?;
243 let flag_show_repeated = call.has_flag(engine_state, stack, "repeated")?;
244 let flag_ignore_case = call.has_flag(engine_state, stack, "ignore-case")?;
245 let flag_only_uniques = call.has_flag(engine_state, stack, "unique")?;
246
247 let signals = engine_state.signals().clone();
248 let uniq_values = input
249 .into_iter()
250 .enumerate()
251 .map_while(|(index, item)| {
252 if signals.interrupted() {
253 return None;
254 }
255 Some(item_mapper(ItemMapperState {
256 item,
257 flag_ignore_case,
258 index,
259 }))
260 })
261 .try_fold(
262 HashMap::<String, ValueCounter>::new(),
263 |mut counter, item| {
264 let key = generate_key(engine_state, &item);
265
266 match key {
267 Ok(key) => {
268 match counter.get_mut(&key) {
269 Some(x) => x.count += 1,
270 None => {
271 counter.insert(key, item);
272 }
273 };
274 Ok(counter)
275 }
276 Err(err) => Err(err),
277 }
278 },
279 );
280
281 let mut uniq_values: HashMap<String, ValueCounter> = uniq_values?;
282
283 if flag_show_repeated {
284 uniq_values.retain(|_v, value_count_pair| value_count_pair.count > 1);
285 }
286
287 if flag_only_uniques {
288 uniq_values.retain(|_v, value_count_pair| value_count_pair.count == 1);
289 }
290
291 let uniq_values = sort(uniq_values.into_iter());
292
293 let result = if flag_show_count {
294 generate_results_with_count(head, uniq_values)
295 } else {
296 uniq_values.into_iter().map(|v| v.val).collect()
297 };
298
299 Ok(Value::list(result, head).into_pipeline_data_with_metadata(metadata))
300}
301
302fn sort(iter: IntoIter<String, ValueCounter>) -> Vec<ValueCounter> {
303 iter.map(|item| item.1)
304 .sorted_by(|a, b| a.index.cmp(&b.index))
305 .collect()
306}
307
308#[cfg(test)]
309mod test {
310 use super::*;
311
312 #[test]
313 fn test_examples() {
314 use crate::test_examples;
315
316 test_examples(Uniq {})
317 }
318}