Skip to content

utils

AvgrageMeter

Bases: object

Computes and stores the average and current value.

Source code in autora/theorist/darts/utils.py
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
class AvgrageMeter(object):
    """
    Computes and stores the average and current value.
    """

    def __init__(self):
        """
        Initializes the average meter.
        """
        self.reset()

    def reset(self):
        """
        Resets the average meter.
        """
        self.avg = 0
        self.sum = 0
        self.cnt = 0

    def update(self, val: float, n: int = 1):
        """
        Updates the average meter.

        Arguments:
            val: value to update the average meter with
            n: number of times to update the average meter
        """
        self.sum += val * n
        self.cnt += n
        self.avg = self.sum / self.cnt

__init__()

Initializes the average meter.

Source code in autora/theorist/darts/utils.py
214
215
216
217
218
def __init__(self):
    """
    Initializes the average meter.
    """
    self.reset()

reset()

Resets the average meter.

Source code in autora/theorist/darts/utils.py
220
221
222
223
224
225
226
def reset(self):
    """
    Resets the average meter.
    """
    self.avg = 0
    self.sum = 0
    self.cnt = 0

update(val, n=1)

Updates the average meter.

Parameters:

Name Type Description Default
val float

value to update the average meter with

required
n int

number of times to update the average meter

1
Source code in autora/theorist/darts/utils.py
228
229
230
231
232
233
234
235
236
237
238
def update(self, val: float, n: int = 1):
    """
    Updates the average meter.

    Arguments:
        val: value to update the average meter with
        n: number of times to update the average meter
    """
    self.sum += val * n
    self.cnt += n
    self.avg = self.sum / self.cnt

accuracy(output, target, topk=(1))

Computes the accuracy over the k top predictions for the specified values of k.

Parameters:

Name Type Description Default
output torch.Tensor

output of the model

required
target torch.Tensor

target of the model

required
topk Tuple

values of k to compute the accuracy at

(1)
Source code in autora/theorist/darts/utils.py
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def accuracy(output: torch.Tensor, target: torch.Tensor, topk: Tuple = (1,)) -> List:
    """
    Computes the accuracy over the k top predictions for the specified values of k.

    Arguments:
        output: output of the model
        target: target of the model
        topk: values of k to compute the accuracy at
    """
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

assign_slurm_instance(slurm_id, arch_weight_decay_list, num_node_list, seed_list)

Determines the meta-search parameters based on the slum job id.

Parameters:

Name Type Description Default
slurm_id int

slurm job id

required
arch_weight_decay_list List

list of weight decay values

required
num_node_list List

list of number of nodes

required
seed_list List

list of seeds

required
Source code in autora/theorist/darts/utils.py
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def assign_slurm_instance(
    slurm_id: int,
    arch_weight_decay_list: List,
    num_node_list: List,
    seed_list: List,
) -> Tuple:
    """
    Determines the meta-search parameters based on the slum job id.

    Arguments:
        slurm_id: slurm job id
        arch_weight_decay_list: list of weight decay values
        num_node_list: list of number of nodes
        seed_list: list of seeds
    """

    seed_id = np.floor(
        slurm_id / (len(num_node_list) * len(arch_weight_decay_list))
    ) % len(seed_list)
    k_id = np.floor(slurm_id / (len(arch_weight_decay_list))) % len(num_node_list)
    weight_decay_id = slurm_id % len(arch_weight_decay_list)

    return (
        arch_weight_decay_list[int(weight_decay_id)],
        int(num_node_list[int(k_id)]),
        int(seed_list[int(seed_id)]),
    )

compute_BIC(output_type, model, input, target)

Returns the Bayesian information criterion for a DARTS model.

Parameters:

Name Type Description Default
output_type ValueType

output type of the dependent variable

required
model torch.nn.Module

model to compute the BIC for

required
input torch.Tensor

input of the model

required
target torch.Tensor

target of the model

required
Source code in autora/theorist/darts/utils.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def compute_BIC(
    output_type: ValueType,
    model: torch.nn.Module,
    input: torch.Tensor,
    target: torch.Tensor,
) -> float:
    """
    Returns the Bayesian information criterion for a DARTS model.

    Arguments:
        output_type: output type of the dependent variable
        model: model to compute the BIC for
        input: input of the model
        target: target of the model
    """

    # compute raw model output
    classifier_output = model(input)

    # compute associated probability
    m = get_output_format(output_type)
    prediction = m(classifier_output).detach()

    k, _, _ = model.countParameters()  # for most likely architecture

    if output_type == ValueType.CLASS:
        target_flattened = torch.flatten(target.long())
        llik = 0
        for idx in range(len(target_flattened)):
            lik = prediction[idx, target_flattened[idx]]
            llik += np.log(lik)
        n = len(target_flattened)  # number of data points

        BIC = np.log(n) * k - 2 * llik
        BIC = BIC

    elif output_type == ValueType.PROBABILITY_SAMPLE:
        llik = 0
        for idx in range(len(target)):

            # fail safe if model doesn't produce probabilities
            if prediction[idx] > 1:
                prediction[idx] = 1
            elif prediction[idx] < 0:
                prediction[idx] = 0

            if target[idx] == 1:
                lik = prediction[idx]
            elif target[idx] == 0:
                lik = 1 - prediction[idx]
            else:
                raise Exception("Target must contain either zeros or ones.")
            llik += np.log(lik)
        n = len(target)  # number of data points

        BIC = np.log(n) * k - 2 * llik
        BIC = BIC[0]

    else:
        raise Exception(
            "BIC computation not implemented for output type "
            + str(ValueType.PROBABILITY)
            + "."
        )

    return BIC

compute_BIC_AIC(soft_targets, soft_prediction, model)

Returns the Bayesian information criterion (BIC) as well as the Aikaike information criterion (AIC) for a DARTS model.

Parameters:

Name Type Description Default
soft_targets np.array

soft target of the model

required
soft_prediction np.array

soft prediction of the model

required
model Network

model to compute the BIC and AIC for

required
Source code in autora/theorist/darts/utils.py
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def compute_BIC_AIC(
    soft_targets: np.array, soft_prediction: np.array, model: Network
) -> Tuple:
    """
    Returns the Bayesian information criterion (BIC) as well as the
    Aikaike information criterion (AIC) for a DARTS model.

    Arguments:
        soft_targets: soft target of the model
        soft_prediction: soft prediction of the model
        model: model to compute the BIC and AIC for
    """

    lik = np.sum(
        np.multiply(soft_prediction, soft_targets), axis=1
    )  # likelihood of data given model
    llik = np.sum(np.log(lik))  # log likelihood
    n = len(lik)  # number of data points
    k, _, _ = model.count_parameters()  # for most likely architecture

    BIC = np.log(n) * k - 2 * llik

    AIC = 2 * k - 2 * llik

    return BIC, AIC

count_parameters_in_MB(model)

Returns the number of parameters for a model.

Parameters:

Name Type Description Default
model Network

model to count the parameters for

required
Source code in autora/theorist/darts/utils.py
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def count_parameters_in_MB(model: Network) -> int:
    """
    Returns the number of parameters for a model.

    Arguments:
        model: model to count the parameters for
    """
    return (
        np.sum(
            np.prod(v.size())
            for name, v in model.named_parameters()
            if "auxiliary" not in name
        )
        / 1e6
    )

create_exp_dir(path, scripts_to_save=None, parent_folder='exps', results_folder=None)

Creates an experiment directory and saves all necessary scripts and files.

Parameters:

Name Type Description Default
path str

path to save the experiment directory to

required
scripts_to_save List

list of scripts to save

None
parent_folder str

parent folder for the experiment directory

'exps'
results_folder str

folder for the results of the experiment

None
Source code in autora/theorist/darts/utils.py
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
def create_exp_dir(
    path: str,
    scripts_to_save: List = None,
    parent_folder: str = "exps",
    results_folder: str = None,
):
    """
    Creates an experiment directory and saves all necessary scripts and files.

    Arguments:
        path: path to save the experiment directory to
        scripts_to_save: list of scripts to save
        parent_folder: parent folder for the experiment directory
        results_folder: folder for the results of the experiment
    """
    os.chdir(parent_folder)  # Edit SM 10/23/19: use local experiment directory
    if not os.path.exists(path):
        os.mkdir(path)
    print("Experiment dir : {}".format(path))

    if results_folder is not None:
        try:
            os.mkdir(os.path.join(path, results_folder))
        except OSError:
            pass

    if scripts_to_save is not None:
        try:
            os.mkdir(os.path.join(path, "scripts"))
        except OSError:
            pass
        os.chdir("..")  # Edit SM 10/23/19: use local experiment directory
        for script in scripts_to_save:
            dst_file = os.path.join(
                parent_folder, path, "scripts", os.path.basename(script)
            )
            shutil.copyfile(script, dst_file)

create_output_file_name(file_prefix, log_version=None, weight_decay=None, k=None, seed=None, theorist=None)

Creates a file name for the output file of a theorist study.

Parameters:

Name Type Description Default
file_prefix str

prefix of the file name

required
log_version int

log version of the theorist run

None
weight_decay float

weight decay of the model

None
k int

number of nodes in the model

None
seed int

seed of the model

None
theorist str

name of the DARTS variant

None
Source code in autora/theorist/darts/utils.py
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def create_output_file_name(
    file_prefix: str,
    log_version: int = None,
    weight_decay: float = None,
    k: int = None,
    seed: int = None,
    theorist: str = None,
) -> str:
    """
    Creates a file name for the output file of a theorist study.

    Arguments:
        file_prefix: prefix of the file name
        log_version: log version of the theorist run
        weight_decay: weight decay of the model
        k: number of nodes in the model
        seed: seed of the model
        theorist: name of the DARTS variant
    """

    output_str = file_prefix

    if theorist is not None:
        output_str += "_" + str(theorist)

    if log_version is not None:
        output_str += "_v_" + str(log_version)

    if weight_decay is not None:
        output_str += "_wd_" + str(weight_decay)

    if k is not None:
        output_str += "_k_" + str(k)

    if k is not None:
        output_str += "_s_" + str(seed)

    return output_str

cross_entropy(pred, soft_targets)

Returns the cross entropy loss for a soft target.

Parameters:

Name Type Description Default
pred torch.Tensor

prediction of the model

required
soft_targets torch.Tensor

soft target of the model

required
Source code in autora/theorist/darts/utils.py
195
196
197
198
199
200
201
202
203
204
205
206
def cross_entropy(pred: torch.Tensor, soft_targets: torch.Tensor) -> torch.Tensor:
    """
    Returns the cross entropy loss for a soft target.

    Arguments:
        pred: prediction of the model
        soft_targets: soft target of the model
    """
    # assuming pred and soft_targets are both Variables with shape (batchsize, num_of_classes),
    # each row of pred is predicted logits and each row of soft_targets is a discrete distribution.
    logsoftmax = nn.LogSoftmax(dim=1)
    return torch.mean(torch.sum(-soft_targets * logsoftmax(pred), 1))

format_input_target(input, target, criterion)

Formats the input and target for the model.

Parameters:

Name Type Description Default
input torch.tensor

input to the model

required
target torch.tensor

target of the model

required
criterion Callable

criterion to use for the model

required

Returns:

Name Type Description
input Tuple[torch.tensor, torch.tensor]

formatted input and target for the model

Source code in autora/theorist/darts/utils.py
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def format_input_target(
    input: torch.tensor, target: torch.tensor, criterion: Callable
) -> Tuple[torch.tensor, torch.tensor]:
    """
    Formats the input and target for the model.

    Args:
        input: input to the model
        target: target of the model
        criterion: criterion to use for the model

    Returns:
        input: formatted input and target for the model

    """

    if isinstance(criterion, nn.CrossEntropyLoss):
        target = target.squeeze()

    return (input, target)

get_best_fitting_models(model_name_list, loss_list, BIC_list, topk)

Returns the topk best fitting models.

Parameters:

Name Type Description Default
model_name_list List

list of model names

required
loss_list List

list of loss values

required
BIC_list List

list of BIC values

required
topk int

number of topk models to return

required
Source code in autora/theorist/darts/utils.py
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
def get_best_fitting_models(
    model_name_list: List,
    loss_list: List,
    BIC_list: List,
    topk: int,
) -> Tuple:
    """
    Returns the topk best fitting models.

    Arguments:
        model_name_list: list of model names
        loss_list: list of loss values
        BIC_list: list of BIC values
        topk: number of topk models to return
    """

    topk_losses = sorted(zip(loss_list, model_name_list), reverse=False)[:topk]
    res = list(zip(*topk_losses))
    topk_losses_names = res[1]

    topk_BICs = sorted(zip(BIC_list, model_name_list), reverse=False)[:topk]
    res = list(zip(*topk_BICs))
    topk_BICs_names = res[1]

    return (topk_losses_names, topk_BICs_names)

get_loss_function(outputType)

Returns the loss function for the given output type of a dependent variable.

Parameters:

Name Type Description Default
outputType ValueType

output type of the dependent variable

required
Source code in autora/theorist/darts/utils.py
440
441
442
443
444
445
446
447
448
def get_loss_function(outputType: ValueType):
    """
    Returns the loss function for the given output type of a dependent variable.

    Arguments:
        outputType: output type of the dependent variable
    """

    return LOSS_FUNCTION_MAPPING.get(outputType, nn.MSELoss())

get_output_format(outputType)

Returns the output format (activation function of the final output layer) for the given output type of a dependent variable.

Parameters:

Name Type Description Default
outputType ValueType

output type of the dependent variable

required
Source code in autora/theorist/darts/utils.py
461
462
463
464
465
466
467
468
469
470
def get_output_format(outputType: ValueType):
    """
    Returns the output format (activation function of the final output layer)
    for the given output type of a dependent variable.

    Arguments:
        outputType: output type of the dependent variable
    """

    return OUTPUT_FORMAT_MAPPING.get(outputType, nn.MSELoss())

get_output_str(outputType)

Returns the output string for the given output type of a dependent variable.

Parameters:

Name Type Description Default
outputType ValueType

output type of the dependent variable

required
Source code in autora/theorist/darts/utils.py
483
484
485
486
487
488
489
490
491
def get_output_str(outputType: ValueType) -> str:
    """
    Returns the output string for the given output type of a dependent variable.

    Arguments:
        outputType: output type of the dependent variable
    """

    return OUTPUT_STR_MAPPING.get(outputType, "")

load(model, model_path)

Loads a model from a file.

Source code in autora/theorist/darts/utils.py
297
298
299
300
301
def load(model: torch.nn.Module, model_path: str):
    """
    Loads a model from a file.
    """
    model.load_state_dict(torch.load(model_path))

read_log_files(results_path, winning_architecture_only=False)

Reads the log files from an experiment directory and returns the results.

Parameters:

Name Type Description Default
results_path str

path to the experiment results directory

required
winning_architecture_only bool

if True, only the winning architecture is returned

False
Source code in autora/theorist/darts/utils.py
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
def read_log_files(results_path: str, winning_architecture_only: bool = False) -> Tuple:
    """
    Reads the log files from an experiment directory and returns the results.

    Arguments:
        results_path: path to the experiment results directory
        winning_architecture_only: if True, only the winning architecture is returned
    """

    current_wd = os.getcwd()

    os.chdir(results_path)
    filelist = glob.glob("*.{}".format("csv"))

    model_name_list = list()
    loss_list = list()
    BIC_list = list()
    AIC_list = list()

    # READ LOG FILES

    print("Reading log files... ")
    for file in filelist:

        with open(file) as csvfile:
            readCSV = csv.reader(csvfile, delimiter=",")
            for row in readCSV:
                if winning_architecture_only is False or "sample0" in row[0]:
                    model_name_list.append(row[0])
                    loss_list.append(float(row[1]))
                    BIC_list.append(float(row[2].replace("[", "").replace("]", "")))
                    AIC_list.append(float(row[3].replace("[", "").replace("]", "")))

    os.chdir(current_wd)

    return (model_name_list, loss_list, BIC_list, AIC_list)

save(model, model_path, exp_folder=None)

Saves a model to a file.

Parameters:

Name Type Description Default
model torch.nn.Module

model to save

required
model_path str

path to save the model to

required
exp_folder str

general experiment directory to save the model to

None
Source code in autora/theorist/darts/utils.py
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def save(model: torch.nn.Module, model_path: str, exp_folder: str = None):
    """
    Saves a model to a file.

    Arguments:
        model: model to save
        model_path: path to save the model to
        exp_folder: general experiment directory to save the model to
    """
    if exp_folder is not None:
        os.chdir("exps")  # Edit SM 10/23/19: use local experiment directory
    torch.save(model.state_dict(), model_path)
    if exp_folder is not None:
        os.chdir("..")  # Edit SM 10/23/19: use local experiment directory

sigmid_mse(output, target)

Returns the MSE loss for a sigmoid output.

Parameters:

Name Type Description Default
output torch.Tensor

output of the model

required
target torch.Tensor

target of the model

required
Source code in autora/theorist/darts/utils.py
84
85
86
87
88
89
90
91
92
93
94
95
def sigmid_mse(output: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
    """
    Returns the MSE loss for a sigmoid output.

    Arguments:
        output: output of the model
        target: target of the model
    """
    m = nn.Sigmoid()
    output = m(output)
    loss = torch.mean((output - target) ** 2)
    return loss