strtok -> strmsep

c/lib/mlrutil.c

@@ -26,6 +26,23 @@ void mlr_internal_coding_error_unless(int v, char* file, int line) {
 	}
 }
 
+// ----------------------------------------------------------------
+char* mlr_strmsep(char **pstring, const char *sep, int seplen) {
+	char* string = *pstring;
+	if (string == NULL) {
+		return NULL;
+	}
+	char* pnext = strstr(string, sep);
+	if (pnext == NULL) {
+		*pstring = NULL;
+		return string;
+	} else {
+		*pnext = 0;
+		*pstring = pnext + seplen;
+		return string;
+	}
+}
+
 // ----------------------------------------------------------------
 int mlr_bsearch_double_for_insert(double* array, int size, double value) {
 	int lo = 0;

c/lib/mlrutil.h

@@ -73,6 +73,11 @@ static inline int streqn(char* a, char* b, int n) {
 #endif
 }
 
+// ----------------------------------------------------------------
+// Like strsep but the sep argument is a multi-character delimiter,
+// not a set of single-character delimiters.
+char* mlr_strmsep(char **pstring, const char *sep, int seplen);
+
 // ----------------------------------------------------------------
 int mlr_bsearch_double_for_insert(double* array, int size, double value);
 

c/mapping/mapper_nest.c

@@ -250,14 +250,18 @@ static sllv_t* mapper_nest_explode_values_across_fields(lrec_t* pinrec, context_
 	lrece_t* porig = pentry;
 
 	char* sep = pstate->nested_fs;
+	int seplen = strlen(sep);
 	int i = 1;
-	for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep), i++) {
+	char* walker = field_value;
+	char* piece = NULL;
+	while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
 		char  istring_free_flags;
 		char* istring = low_int_to_string(i, &istring_free_flags);
 		char* new_key = mlr_paste_3_strings(pstate->field_name, "_", istring);
 		if (istring_free_flags & FREE_ENTRY_KEY)
 			free(istring);
 		pentry = lrec_put_after(pinrec, pentry, new_key, mlr_strdup_or_die(piece), FREE_ENTRY_KEY|FREE_ENTRY_VALUE);
+		i++;
 	}
 	lrec_unlink_and_free(pinrec, porig);
 	return sllv_single(pinrec);;
@@ -314,7 +318,10 @@ static sllv_t* mapper_nest_explode_values_across_records(lrec_t* pinrec, context
 
 	sllv_t* poutrecs = sllv_alloc();
 	char* sep = pstate->nested_fs;
-	for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) {
+	int seplen = strlen(sep);
+	char* walker = field_value;
+	char* piece = NULL;
+	while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
 		lrec_t* poutrec = lrec_copy(pinrec);
 		lrec_put(poutrec, pstate->field_name, mlr_strdup_or_die(piece), FREE_ENTRY_VALUE);
 		sllv_append(poutrecs, poutrec);
@@ -400,7 +407,10 @@ static sllv_t* mapper_nest_explode_pairs_across_fields(lrec_t* pinrec, context_t
 	lrece_t* porig = pentry;
 
 	char* sep = pstate->nested_fs;
-	for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) {
+	int seplen = strlen(sep);
+	char* walker = field_value;
+	char* piece = NULL;
+	while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
 		char* found_sep = strstr(piece, pstate->nested_ps);
 		if (found_sep != NULL) { // there is a pair
 			*found_sep = 0;
@@ -430,7 +440,10 @@ static sllv_t* mapper_nest_explode_pairs_across_records(lrec_t* pinrec, context_
 
 	sllv_t* poutrecs = sllv_alloc();
 	char* sep = pstate->nested_fs;
-	for (char* piece = strtok(field_value, sep); piece != NULL; piece = strtok(NULL, sep)) {
+	int seplen = strlen(sep);
+	char* walker = field_value;
+	char* piece = NULL;
+	while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
 		char* found_sep = strstr(piece, pstate->nested_ps);
 		lrec_t* poutrec = lrec_copy(pinrec);
 		lrece_t* pe = NULL;

c/output/lrec_writer_json.c

@@ -117,15 +117,18 @@ static void lrec_writer_json_process(void* pvstate, FILE* output_stream, lrec_t*
 		mlhmmv_root_t* pmap = mlhmmv_root_alloc();
 
 		char* sep = pstate->output_json_flatten_separator;
+		int seplen = strlen(sep);
 
 		for (lrece_t* pe = prec->phead; pe != NULL; pe = pe->pnext) {
-			// strdup since strtok is destructive and CSV/PPRINT header fields
+			// strdup since strmsep is destructive and CSV/PPRINT header fields
 			// are shared across multiple records
 			char* lkey = mlr_strdup_or_die(pe->key);
 			char* lvalue = pe->value;
 
 			sllmv_t* pmvkeys = sllmv_alloc();
-			for (char* piece = strtok(lkey, sep); piece != NULL; piece = strtok(NULL, sep)) {
+			char* walker = lkey;
+			char* piece = NULL;
+			while ((piece = mlr_strmsep(&walker, sep, seplen)) != NULL) {
 				mv_t mvkey = mv_from_string(piece, NO_FREE);
 				sllmv_append_no_free(pmvkeys, &mvkey);
 			}

c/reg_test/expected/out

@@ -3262,25 +3262,25 @@ x=a=4|b=5,y=d=70
 
 mlr nest --explode --values --across-fields -f x ./reg_test/input/nest-explode.dkvp
 x_1=a:1,x_2=b:2,x_3=c:3,y=d:40
-y=d:50
+x_1=,y=d:50
 u=100,y=d:60
 x_1=a:4,x_2=b:5,y=d:70
 
 mlr nest --explode --values --across-fields -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp
 x_1=a=1,x_2=b=2,x_3=c=3,y=d=40
-y=d=50
+x_1=,y=d=50
 u=100,y=d=60
 x_1=a=4,x_2=b=5,y=d=70
 
 mlr nest --explode --values --across-fields -f x then nest --implode --values --across-fields -f x ./reg_test/input/nest-explode.dkvp
 x=a:1;b:2;c:3,y=d:40
-y=d:50
+x=,y=d:50
 u=100,y=d:60
 x=a:4;b:5,y=d:70
 
 mlr nest --explode --values --across-fields -f x --nested-fs pipe --nested-ps = then nest --implode --values --across-fields -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp
 x=a=1|b=2|c=3,y=d=40
-y=d=50
+x=,y=d=50
 u=100,y=d=60
 x=a=4|b=5,y=d=70
 
@@ -3288,6 +3288,7 @@ mlr nest --explode --values --across-records -f x ./reg_test/input/nest-explode.
 x=a:1,y=d:40
 x=b:2,y=d:40
 x=c:3,y=d:40
+x=,y=d:50
 u=100,y=d:60
 x=a:4,y=d:70
 x=b:5,y=d:70
@@ -3296,6 +3297,7 @@ mlr nest --explode --values --across-records -f x --nested-fs pipe --nested-ps =
 x=a=1,y=d=40
 x=b=2,y=d=40
 x=c=3,y=d=40
+x=,y=d=50
 u=100,y=d=60
 x=a=4,y=d=70
 x=b=5,y=d=70
@@ -3303,11 +3305,13 @@ x=b=5,y=d=70
 mlr nest --explode --values --across-records -f x then nest --implode --values --across-records -f x ./reg_test/input/nest-explode.dkvp
 u=100,y=d:60
 x=a:1;b:2;c:3,y=d:40
+x=,y=d:50
 x=a:4;b:5,y=d:70
 
 mlr nest --explode --values --across-records -f x --nested-fs pipe --nested-ps = then nest --implode --values --across-records -f x --nested-fs pipe --nested-ps = ./reg_test/input/nest-explode-vary-fs-ps.dkvp
 u=100,y=d=60
 x=a=1|b=2|c=3,y=d=40
+x=,y=d=50
 x=a=4|b=5,y=d=70
 
 mlr nest --explode --pairs --across-fields -f x ./reg_test/input/nest-explode.dkvp
@@ -3326,6 +3330,7 @@ mlr nest --explode --pairs --across-records -f x ./reg_test/input/nest-explode.d
 a=1,y=d:40
 b=2,y=d:40
 c=3,y=d:40
+y=d:50
 u=100,y=d:60
 a=4,y=d:70
 b=5,y=d:70
@@ -3334,6 +3339,7 @@ mlr nest --explode --pairs --across-records -f x --nested-fs pipe --nested-ps =
 a=1,y=d=40
 b=2,y=d=40
 c=3,y=d=40
+y=d=50
 u=100,y=d=60
 a=4,y=d=70
 b=5,y=d=70

c/todo.txt

@@ -15,10 +15,13 @@ BUGFIXES
 * xtab.wtf
 ? tsv->json blank line -- ? needs repro.
 
+* nidx with repifs & leading spaces :^/
+
 ================================================================
 5.0.0 POST-RELEASE TO DO:
 
 https://github.com/johnkerl/miller/issues/132
+-> strmsep UTs
 
 ! cook:
 
@@ -296,6 +299,8 @@ COMPARES:
 * mlr paste (sjackman)
 * dump @records after @records[NR]=$*: json "1" rather than 1 indices are b04k3d. make a cliopt.
 
+* regexes for IPS/IFS/IRS; also in mlr nest etc.
+
 !!?? disallow implicit assignment? force w/ var at least??
   - w/  implicit: for (...) {sum += x} FUBAR
   - w/o implicit: sad panda cannot type 'x=1'