Upload dict_utils.py with huggingface_hub
Browse files- dict_utils.py +97 -56
dict_utils.py
CHANGED
@@ -178,16 +178,24 @@ def dict_delete(
|
|
178 |
):
|
179 |
# We remove from dic the value from each and every element lead to by a path matching the query.
|
180 |
# If remove_empty_ancestors=True, and the removal of any such value leaves its containing element (list or dict)
|
181 |
-
# within dic empty -- remove that element as well, and continue recursively
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
182 |
qpath = validate_query_and_break_to_components(query)
|
183 |
-
if len(qpath) == 1:
|
184 |
-
if qpath[0] in dic:
|
185 |
-
dic.pop(qpath[0])
|
186 |
-
return
|
187 |
-
if not not_exist_ok:
|
188 |
-
raise ValueError(
|
189 |
-
f"An attempt to delete from dictionary {dic}, an element {query}, that does not exist in the dictionary"
|
190 |
-
)
|
191 |
|
192 |
try:
|
193 |
success, new_val = delete_values(
|
@@ -196,10 +204,17 @@ def dict_delete(
|
|
196 |
index_into_query=(-1) * len(qpath),
|
197 |
remove_empty_ancestors=remove_empty_ancestors,
|
198 |
)
|
199 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
200 |
raise ValueError(
|
201 |
-
f"An attempt to delete from dictionary {dic}, an element {query}, that does not exist in the dictionary"
|
202 |
)
|
|
|
203 |
except Exception as e:
|
204 |
raise ValueError(f"query {query} matches no path in dictionary {dic}") from e
|
205 |
|
@@ -233,7 +248,8 @@ def get_values(
|
|
233 |
except:
|
234 |
continue
|
235 |
|
236 |
-
return (len(to_ret) > 0, to_ret)
|
|
|
237 |
# next_component is indx or name, current_element must be a list or a dict
|
238 |
if indx.match(component):
|
239 |
component = int(component)
|
@@ -261,15 +277,15 @@ def set_values(
|
|
261 |
return (True, value) # matched query all along!
|
262 |
|
263 |
# current_element should be a list or dict: a containing element
|
264 |
-
if current_element and not isinstance(current_element, (list, dict)):
|
265 |
current_element = None # give it a chance to become what is needed, if allowed
|
266 |
|
267 |
-
if
|
268 |
return (False, None)
|
269 |
-
component = fixed_parameters["query"][index_into_query]
|
270 |
|
|
|
271 |
if component == "*":
|
272 |
-
if current_element and set_multiple:
|
273 |
if isinstance(current_element, dict) and len(current_element) != len(value):
|
274 |
return (False, None)
|
275 |
if isinstance(current_element, list) and len(current_element) > len(value):
|
@@ -279,8 +295,15 @@ def set_values(
|
|
279 |
return (False, None)
|
280 |
# current_element must be a list, extend current_element to the length needed
|
281 |
current_element.extend([None] * (len(value) - len(current_element)))
|
282 |
-
if
|
283 |
-
current_element = [None] * (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
284 |
# now current_element is of size suiting value
|
285 |
if isinstance(current_element, dict):
|
286 |
keys = sorted(current_element.keys())
|
@@ -304,32 +327,30 @@ def set_values(
|
|
304 |
|
305 |
except:
|
306 |
continue
|
307 |
-
return (
|
|
|
|
|
|
|
308 |
|
309 |
# component is an index into a list or a key into a dictionary
|
310 |
if indx.match(component):
|
311 |
-
if current_element
|
312 |
if not fixed_parameters["generate_if_not_exists"]:
|
313 |
return (False, None)
|
314 |
current_element = []
|
|
|
315 |
component = int(component)
|
316 |
-
if (
|
317 |
-
current_element and component >= len(current_element)
|
318 |
-
) or not current_element:
|
319 |
if not fixed_parameters["generate_if_not_exists"]:
|
320 |
return (False, None)
|
321 |
# extend current_element to the length needed
|
322 |
-
if not current_element:
|
323 |
-
current_element = []
|
324 |
current_element.extend([None] * (component + 1 - len(current_element)))
|
325 |
next_current_element = current_element[component]
|
326 |
else: # component is a key into a dictionary
|
327 |
-
if current_element
|
328 |
if not fixed_parameters["generate_if_not_exists"]:
|
329 |
return (False, None)
|
330 |
current_element = {}
|
331 |
-
if not current_element:
|
332 |
-
current_element = {}
|
333 |
if (
|
334 |
component not in current_element
|
335 |
and not fixed_parameters["generate_if_not_exists"]
|
@@ -358,12 +379,20 @@ def set_values(
|
|
358 |
def dict_get(
|
359 |
dic: dict,
|
360 |
query: str,
|
361 |
-
use_dpath: bool = True,
|
362 |
not_exist_ok: bool = False,
|
363 |
default: Any = None,
|
364 |
):
|
365 |
-
if
|
366 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
367 |
try:
|
368 |
success, values = get_values(dic, components, -1 * len(components))
|
369 |
if not success:
|
@@ -372,12 +401,6 @@ def dict_get(
|
|
372 |
raise ValueError(
|
373 |
f'query "{query}" did not match any item in dict: {dic}'
|
374 |
)
|
375 |
-
if isinstance(values, list) and len(values) == 0:
|
376 |
-
if not_exist_ok:
|
377 |
-
return default
|
378 |
-
raise ValueError(
|
379 |
-
f'query "{query}" did not match any item in dict: {dic} while not_exist_ok=False'
|
380 |
-
)
|
381 |
|
382 |
return values
|
383 |
|
@@ -385,18 +408,17 @@ def dict_get(
|
|
385 |
if not_exist_ok:
|
386 |
return default
|
387 |
raise ValueError(
|
388 |
-
f'query "{query}" did not match any item in dict: {dic}
|
389 |
) from e
|
390 |
|
391 |
-
|
392 |
-
|
|
|
393 |
|
394 |
if not_exist_ok:
|
395 |
return default
|
396 |
|
397 |
-
raise ValueError(
|
398 |
-
f'query "{query}" did not match any item in dict: {dic} while not_exist_ok=False'
|
399 |
-
)
|
400 |
|
401 |
|
402 |
# dict_set sets a value, 'value', which by itself, can be a dict or list or scalar, into 'dic', to become the value of
|
@@ -448,25 +470,44 @@ def dict_set(
|
|
448 |
dic: dict,
|
449 |
query: str,
|
450 |
value: Any,
|
451 |
-
use_dpath=True,
|
452 |
not_exist_ok=True,
|
453 |
set_multiple=False,
|
454 |
-
):
|
455 |
-
if
|
456 |
-
not isinstance(value, list) or len(value) == 0 or "*" not in query
|
457 |
-
):
|
458 |
raise ValueError(
|
459 |
-
f"
|
460 |
)
|
461 |
-
|
462 |
-
|
463 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
464 |
return
|
465 |
-
raise ValueError(
|
466 |
-
f"not_exist_ok=False and the single component query '{query}' is not a key in dic {dic}"
|
467 |
-
)
|
468 |
|
469 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
470 |
components = validate_query_and_break_to_components(query)
|
471 |
fixed_parameters = {
|
472 |
"query": components,
|
|
|
178 |
):
|
179 |
# We remove from dic the value from each and every element lead to by a path matching the query.
|
180 |
# If remove_empty_ancestors=True, and the removal of any such value leaves its containing element (list or dict)
|
181 |
+
# within dic empty -- remove that element as well, and continue recursively, but stop one step before deleting dic
|
182 |
+
# altogether, even if became {}. If successful, changes dic into its new shape
|
183 |
+
|
184 |
+
if dic is None or not isinstance(dic, (list, dict)):
|
185 |
+
raise ValueError(
|
186 |
+
f"dic {dic} is either None or not a list nor a dict. Can not delete from it."
|
187 |
+
)
|
188 |
+
|
189 |
+
if len(query) == 0:
|
190 |
+
raise ValueError(
|
191 |
+
"Query is an empty string, implying the deletion of dic as a whole. This can not be done via this function call."
|
192 |
+
)
|
193 |
+
|
194 |
+
if isinstance(dic, dict) and query.strip() in dic:
|
195 |
+
dic.pop(query.strip())
|
196 |
+
return
|
197 |
+
|
198 |
qpath = validate_query_and_break_to_components(query)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
199 |
|
200 |
try:
|
201 |
success, new_val = delete_values(
|
|
|
204 |
index_into_query=(-1) * len(qpath),
|
205 |
remove_empty_ancestors=remove_empty_ancestors,
|
206 |
)
|
207 |
+
|
208 |
+
if success:
|
209 |
+
if new_val == {}:
|
210 |
+
dic.clear()
|
211 |
+
return
|
212 |
+
|
213 |
+
if not not_exist_ok:
|
214 |
raise ValueError(
|
215 |
+
f"An attempt to delete from dictionary {dic}, an element {query}, that does not exist in the dictionary, while not_exist_ok=False"
|
216 |
)
|
217 |
+
|
218 |
except Exception as e:
|
219 |
raise ValueError(f"query {query} matches no path in dictionary {dic}") from e
|
220 |
|
|
|
248 |
except:
|
249 |
continue
|
250 |
|
251 |
+
return (len(to_ret) > 0 or index_into_query == -1, to_ret)
|
252 |
+
# when * is the last component, it refers to 'all the contents' of an empty list being current_element.
|
253 |
# next_component is indx or name, current_element must be a list or a dict
|
254 |
if indx.match(component):
|
255 |
component = int(component)
|
|
|
277 |
return (True, value) # matched query all along!
|
278 |
|
279 |
# current_element should be a list or dict: a containing element
|
280 |
+
if current_element is not None and not isinstance(current_element, (list, dict)):
|
281 |
current_element = None # give it a chance to become what is needed, if allowed
|
282 |
|
283 |
+
if current_element is None and not fixed_parameters["generate_if_not_exists"]:
|
284 |
return (False, None)
|
|
|
285 |
|
286 |
+
component = fixed_parameters["query"][index_into_query]
|
287 |
if component == "*":
|
288 |
+
if current_element is not None and set_multiple:
|
289 |
if isinstance(current_element, dict) and len(current_element) != len(value):
|
290 |
return (False, None)
|
291 |
if isinstance(current_element, list) and len(current_element) > len(value):
|
|
|
295 |
return (False, None)
|
296 |
# current_element must be a list, extend current_element to the length needed
|
297 |
current_element.extend([None] * (len(value) - len(current_element)))
|
298 |
+
if current_element is None or current_element == []:
|
299 |
+
current_element = [None] * (
|
300 |
+
len(value)
|
301 |
+
if set_multiple
|
302 |
+
else value is None
|
303 |
+
or not isinstance(value, list)
|
304 |
+
or len(value) > 0
|
305 |
+
or index_into_query < -1
|
306 |
+
)
|
307 |
# now current_element is of size suiting value
|
308 |
if isinstance(current_element, dict):
|
309 |
keys = sorted(current_element.keys())
|
|
|
327 |
|
328 |
except:
|
329 |
continue
|
330 |
+
return (
|
331 |
+
any_success or (len(keys) == 0 and index_into_query == -1),
|
332 |
+
current_element,
|
333 |
+
)
|
334 |
|
335 |
# component is an index into a list or a key into a dictionary
|
336 |
if indx.match(component):
|
337 |
+
if current_element is None or not isinstance(current_element, list):
|
338 |
if not fixed_parameters["generate_if_not_exists"]:
|
339 |
return (False, None)
|
340 |
current_element = []
|
341 |
+
# current_element is a list
|
342 |
component = int(component)
|
343 |
+
if component >= len(current_element):
|
|
|
|
|
344 |
if not fixed_parameters["generate_if_not_exists"]:
|
345 |
return (False, None)
|
346 |
# extend current_element to the length needed
|
|
|
|
|
347 |
current_element.extend([None] * (component + 1 - len(current_element)))
|
348 |
next_current_element = current_element[component]
|
349 |
else: # component is a key into a dictionary
|
350 |
+
if current_element is None or not isinstance(current_element, dict):
|
351 |
if not fixed_parameters["generate_if_not_exists"]:
|
352 |
return (False, None)
|
353 |
current_element = {}
|
|
|
|
|
354 |
if (
|
355 |
component not in current_element
|
356 |
and not fixed_parameters["generate_if_not_exists"]
|
|
|
379 |
def dict_get(
|
380 |
dic: dict,
|
381 |
query: str,
|
|
|
382 |
not_exist_ok: bool = False,
|
383 |
default: Any = None,
|
384 |
):
|
385 |
+
if len(query.strip()) == 0:
|
386 |
+
return dic
|
387 |
+
|
388 |
+
if dic is None:
|
389 |
+
raise ValueError("Can not get any value from a dic that is None")
|
390 |
+
|
391 |
+
if isinstance(dic, dict) and query.strip() in dic:
|
392 |
+
return dic[query.strip()]
|
393 |
+
|
394 |
+
components = validate_query_and_break_to_components(query)
|
395 |
+
if len(components) > 1:
|
396 |
try:
|
397 |
success, values = get_values(dic, components, -1 * len(components))
|
398 |
if not success:
|
|
|
401 |
raise ValueError(
|
402 |
f'query "{query}" did not match any item in dict: {dic}'
|
403 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
404 |
|
405 |
return values
|
406 |
|
|
|
408 |
if not_exist_ok:
|
409 |
return default
|
410 |
raise ValueError(
|
411 |
+
f'query "{query}" did not match any item in dict: {dic}'
|
412 |
) from e
|
413 |
|
414 |
+
# len(components) == 1
|
415 |
+
if components[0] in dic:
|
416 |
+
return dic[components[0]]
|
417 |
|
418 |
if not_exist_ok:
|
419 |
return default
|
420 |
|
421 |
+
raise ValueError(f'query "{query}" did not match any item in dict: {dic}')
|
|
|
|
|
422 |
|
423 |
|
424 |
# dict_set sets a value, 'value', which by itself, can be a dict or list or scalar, into 'dic', to become the value of
|
|
|
470 |
dic: dict,
|
471 |
query: str,
|
472 |
value: Any,
|
|
|
473 |
not_exist_ok=True,
|
474 |
set_multiple=False,
|
475 |
+
): # sets dic to its new value
|
476 |
+
if dic is None or not isinstance(dic, (list, dict)):
|
|
|
|
|
477 |
raise ValueError(
|
478 |
+
f"Can not change dic that is either None or not a dict nor a list. Got dic = {dic}"
|
479 |
)
|
480 |
+
|
481 |
+
if query.strip() == "":
|
482 |
+
# change the whole input dic, as dic indeed matches ""
|
483 |
+
if isinstance(dic, dict):
|
484 |
+
if value is None or not isinstance(value, dict):
|
485 |
+
raise ValueError(
|
486 |
+
f"Through an empty query, trying to set a whole new value, {value}, to the whole of dic, {dic}, but value is not a dict"
|
487 |
+
)
|
488 |
+
dic.clear()
|
489 |
+
dic.update(value)
|
490 |
return
|
|
|
|
|
|
|
491 |
|
492 |
+
if isinstance(dic, list):
|
493 |
+
if value is None or not isinstance(value, list):
|
494 |
+
raise ValueError(
|
495 |
+
f"Through an empty query, trying to set a whole new value, {value}, to the whole of dic, {dic}, but value is not a list"
|
496 |
+
)
|
497 |
+
dic.clear()
|
498 |
+
dic.extend(value)
|
499 |
+
return
|
500 |
+
|
501 |
+
if isinstance(dic, dict) and query.strip() in dic:
|
502 |
+
dic[query.strip()] = value
|
503 |
+
return
|
504 |
+
|
505 |
+
if set_multiple:
|
506 |
+
if value is None or not isinstance(value, list) or len(value) == 0:
|
507 |
+
raise ValueError(
|
508 |
+
f"set_multiple=True, but value, {value}, can not be broken up, as either it is not a list or it is an empty list"
|
509 |
+
)
|
510 |
+
|
511 |
components = validate_query_and_break_to_components(query)
|
512 |
fixed_parameters = {
|
513 |
"query": components,
|