Visualising experiments¶
This tutorial introduces the built-in plotting utilities defined in plots.py. These functions help you inspect and compare the results of a BLADE experiment.
We assume you have already executed an experiment and saved the results using an ExperimentLogger. Replace results/simple_exp with your own experiment folder.
[1]:
from iohblade.loggers import ExperimentLogger
from iohblade.plots import plot_convergence, plot_experiment_CEG, plot_code_evolution_graphs, plot_boxplot_fitness, plot_boxplot_fitness_hue, fitness_table, plot_token_usage
logger = ExperimentLogger('results/simple_exp')
Convergence plot¶
plot_convergence shows the average best fitness over time for all methods on each problem.
[2]:
plot_convergence(logger)
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[2], line 1
----> 1 plot_convergence(logger)
File ~/work/BLADE/BLADE/iohblade/plots.py:189, in plot_convergence(logger, metric, aggregation, variance_aggregation, methods, budget, save, return_fig, separate_lines, show_std)
186 methods_to_use = methods
187 methods, problems = logger.get_methods_problems()
--> 189 fig, axes = plt.subplots(
190 figsize=(8, 6 * len(problems)), nrows=len(problems), ncols=1
191 )
192 problem_i = 0
193 for problem in problems:
194 # Ensure the data is sorted by 'id' and 'fitness'
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/matplotlib/pyplot.py:1887, in subplots(nrows, ncols, sharex, sharey, squeeze, width_ratios, height_ratios, subplot_kw, gridspec_kw, **fig_kw)
1884 _raise_if_figure_exists(fig_kw.get('num'), "subplots", fig_kw.get('clear'))
1886 fig = figure(**fig_kw)
-> 1887 axs = fig.subplots(nrows=nrows, ncols=ncols, sharex=sharex, sharey=sharey,
1888 squeeze=squeeze, subplot_kw=subplot_kw,
1889 gridspec_kw=gridspec_kw, height_ratios=height_ratios,
1890 width_ratios=width_ratios)
1891 return fig, axs
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/matplotlib/figure.py:925, in FigureBase.subplots(self, nrows, ncols, sharex, sharey, squeeze, width_ratios, height_ratios, subplot_kw, gridspec_kw)
921 raise ValueError("'width_ratios' must not be defined both as "
922 "parameter and as key in 'gridspec_kw'")
923 gridspec_kw['width_ratios'] = width_ratios
--> 925 gs = self.add_gridspec(nrows, ncols, figure=self, **gridspec_kw)
926 axs = gs.subplots(sharex=sharex, sharey=sharey, squeeze=squeeze,
927 subplot_kw=subplot_kw)
928 return axs
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/matplotlib/figure.py:1608, in FigureBase.add_gridspec(self, nrows, ncols, **kwargs)
1565 """
1566 Low-level API for creating a `.GridSpec` that has this figure as a parent.
1567
(...) 1604
1605 """
1607 _ = kwargs.pop('figure', None) # pop in case user has added this...
-> 1608 gs = GridSpec(nrows=nrows, ncols=ncols, figure=self, **kwargs)
1609 return gs
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/matplotlib/gridspec.py:364, in GridSpec.__init__(self, nrows, ncols, figure, left, bottom, right, top, wspace, hspace, width_ratios, height_ratios)
361 self.hspace = hspace
362 self.figure = figure
--> 364 super().__init__(nrows, ncols,
365 width_ratios=width_ratios,
366 height_ratios=height_ratios)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/matplotlib/gridspec.py:49, in GridSpecBase.__init__(self, nrows, ncols, height_ratios, width_ratios)
34 """
35 Parameters
36 ----------
(...) 46 If not given, all rows will have the same height.
47 """
48 if not isinstance(nrows, Integral) or nrows <= 0:
---> 49 raise ValueError(
50 f"Number of rows must be a positive integer, not {nrows!r}")
51 if not isinstance(ncols, Integral) or ncols <= 0:
52 raise ValueError(
53 f"Number of columns must be a positive integer, not {ncols!r}")
ValueError: Number of rows must be a positive integer, not 0
<Figure size 800x0 with 0 Axes>
Code evolution graphs (CEG)¶
plot_experiment_CEG visualises how code evolves during optimisation. Each run is plotted separately.
[3]:
plot_experiment_CEG(logger)
plot_code_evolution_graphs is the lower level function used by plot_experiment_CEG. It plots a single run with various complexity metrics.
[4]:
run_data = logger.get_data().query('method_name == "LLaMEA" and seed == 1')
plot_code_evolution_graphs(run_data, save=False)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/scope.py:231, in Scope.resolve(self, key, is_local)
230 if self.has_resolvers:
--> 231 return self.resolvers[key]
233 # if we're here that means that we have no locals and we also have
234 # no resolvers
File /opt/hostedtoolcache/Python/3.11.16/x64/lib/python3.11/collections/__init__.py:1006, in ChainMap.__getitem__(self, key)
1005 pass
-> 1006 return self.__missing__(key)
File /opt/hostedtoolcache/Python/3.11.16/x64/lib/python3.11/collections/__init__.py:998, in ChainMap.__missing__(self, key)
997 def __missing__(self, key):
--> 998 raise KeyError(key)
KeyError: 'method_name'
During handling of the above exception, another exception occurred:
KeyError Traceback (most recent call last)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/scope.py:242, in Scope.resolve(self, key, is_local)
238 try:
239 # last ditch effort we look in temporaries
240 # these are created when parsing indexing expressions
241 # e.g., df[df > 0]
--> 242 return self.temps[key]
243 except KeyError as err:
KeyError: 'method_name'
The above exception was the direct cause of the following exception:
UndefinedVariableError Traceback (most recent call last)
Cell In[4], line 1
----> 1 run_data = logger.get_data().query('method_name == "LLaMEA" and seed == 1')
2 plot_code_evolution_graphs(run_data, save=False)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/frame.py:4834, in DataFrame.query(self, expr, inplace, **kwargs)
4830 msg = f"expr must be a string to be evaluated, {type(expr)} given"
4831 raise ValueError(msg)
4832 kwargs["level"] = kwargs.pop("level", 0) + 1
4833 kwargs["target"] = None
-> 4834 res = self.eval(expr, **kwargs)
4835
4836 try:
4837 result = self.loc[res]
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/frame.py:4960, in DataFrame.eval(self, expr, inplace, **kwargs)
4956 if "target" not in kwargs:
4957 kwargs["target"] = self
4958 kwargs["resolvers"] = tuple(kwargs.get("resolvers", ())) + resolvers
4959
-> 4960 return _eval(expr, inplace=inplace, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/eval.py:339, in eval(expr, parser, engine, local_dict, global_dict, resolvers, level, target, inplace)
330 # get our (possibly passed-in) scope
331 env = ensure_scope(
332 level + 1,
333 global_dict=global_dict,
(...) 336 target=target,
337 )
--> 339 parsed_expr = Expr(expr, engine=engine, parser=parser, env=env)
341 if engine == "numexpr" and (
342 (
343 is_extension_array_dtype(parsed_expr.terms.return_type)
(...) 350 )
351 ):
352 warnings.warn(
353 "Engine has switched to 'python' because numexpr does not support "
354 "extension array dtypes. Please set your engine to python manually.",
355 RuntimeWarning,
356 stacklevel=find_stack_level(),
357 )
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:809, in Expr.__init__(self, expr, engine, parser, env, level)
807 self.parser = parser
808 self._visitor = PARSERS[parser](self.env, self.engine, self.parser)
--> 809 self.terms = self.parse()
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:828, in Expr.parse(self)
824 def parse(self):
825 """
826 Parse an expression.
827 """
--> 828 return self._visitor.visit(self.expr)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:419, in BaseExprVisitor.visit_Module(self, node, **kwargs)
417 raise SyntaxError("only a single expression is allowed")
418 expr = node.body[0]
--> 419 return self.visit(expr, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:422, in BaseExprVisitor.visit_Expr(self, node, **kwargs)
421 def visit_Expr(self, node, **kwargs):
--> 422 return self.visit(node.value, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:746, in BaseExprVisitor.visit_BoolOp(self, node, **kwargs)
743 return self._maybe_evaluate_binop(op, node.op, lhs, rhs)
745 operands = node.values
--> 746 return reduce(visitor, operands)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:739, in BaseExprVisitor.visit_BoolOp.<locals>.visitor(x, y)
738 def visitor(x, y):
--> 739 lhs = self._try_visit_binop(x)
740 rhs = self._try_visit_binop(y)
742 op, op_class, lhs, rhs = self._maybe_transform_eq_ne(node, lhs, rhs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:735, in BaseExprVisitor._try_visit_binop(self, bop)
733 if isinstance(bop, (Op, Term)):
734 return bop
--> 735 return self.visit(bop)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:719, in BaseExprVisitor.visit_Compare(self, node, **kwargs)
717 op = self.translate_In(ops[0])
718 binop = ast.BinOp(op=op, left=node.left, right=comps[0])
--> 719 return self.visit(binop)
721 # recursive case: we have a chained comparison, a CMP b CMP c, etc.
722 left = node.left
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:535, in BaseExprVisitor.visit_BinOp(self, node, **kwargs)
534 def visit_BinOp(self, node, **kwargs):
--> 535 op, op_class, left, right = self._maybe_transform_eq_ne(node)
536 left, right = self._maybe_downcast_constants(left, right)
537 return self._maybe_evaluate_binop(op, op_class, left, right)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:453, in BaseExprVisitor._maybe_transform_eq_ne(self, node, left, right)
451 def _maybe_transform_eq_ne(self, node, left=None, right=None):
452 if left is None:
--> 453 left = self.visit(node.left, side="left")
454 if right is None:
455 right = self.visit(node.right, side="right")
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:413, in BaseExprVisitor.visit(self, node, **kwargs)
411 method = f"visit_{type(node).__name__}"
412 visitor = getattr(self, method)
--> 413 return visitor(node, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/expr.py:545, in BaseExprVisitor.visit_Name(self, node, **kwargs)
544 def visit_Name(self, node, **kwargs) -> Term:
--> 545 return self.term_type(node.id, self.env, **kwargs)
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/ops.py:91, in Term.__init__(self, name, env, side, encoding)
89 tname = str(name)
90 self.is_local = tname.startswith(LOCAL_TAG) or tname in DEFAULT_GLOBALS
---> 91 self._value = self._resolve_name()
92 self.encoding = encoding
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/ops.py:115, in Term._resolve_name(self)
110 if local_name in self.env.scope and isinstance(
111 self.env.scope[local_name], type
112 ):
113 is_local = False
--> 115 res = self.env.resolve(local_name, is_local=is_local)
116 self.update(res)
118 if hasattr(res, "ndim") and res.ndim > 2:
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/computation/scope.py:244, in Scope.resolve(self, key, is_local)
242 return self.temps[key]
243 except KeyError as err:
--> 244 raise UndefinedVariableError(key, is_local) from err
UndefinedVariableError: name 'method_name' is not defined
Boxplots¶
Use plot_boxplot_fitness or plot_boxplot_fitness_hue to compare final fitness across methods or problems.
[5]:
plot_boxplot_fitness(logger)
plot_boxplot_fitness_hue(logger)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In[5], line 1
----> 1 plot_boxplot_fitness(logger)
2 plot_boxplot_fitness_hue(logger)
File ~/work/BLADE/BLADE/iohblade/plots.py:902, in plot_boxplot_fitness(logger, y_label, x_label, problems)
900 # If not already present, create a "fitness" column from the "solution" dictionary
901 if "fitness" not in df.columns:
--> 902 df["fitness"] = df["solution"].apply(
903 lambda sol: sol.get("fitness", float("nan"))
904 )
906 if problems is None:
907 problems = sorted(df["problem_name"].unique())
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/frame.py:4113, in DataFrame.__getitem__(self, key)
4109
4110 if is_single_key:
4111 if self.columns.nlevels > 1:
4112 return self._getitem_multilevel(key)
-> 4113 indexer = self.columns.get_loc(key)
4114 if is_integer(indexer):
4115 indexer = [indexer]
4116 else:
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/indexes/range.py:417, in RangeIndex.get_loc(self, key)
415 raise KeyError(key) from err
416 if isinstance(key, Hashable):
--> 417 raise KeyError(key)
418 self._check_indexing_error(key)
419 raise KeyError(key)
KeyError: 'solution'
Fitness table¶
fitness_table creates a LaTeX table summarising mean and standard deviation per method/problem.
[6]:
fitness_table(logger)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In[6], line 1
----> 1 fitness_table(logger)
File ~/work/BLADE/BLADE/iohblade/plots.py:1006, in fitness_table(logger, alpha, smaller_is_better)
1004 df = logger.get_data().copy()
1005 if "fitness" not in df.columns:
-> 1006 df["fitness"] = df["solution"].apply(
1007 lambda sol: sol.get("fitness", float("nan"))
1008 )
1010 # Group data by (problem_name, method_name)
1011 # We'll store the runs for each combination so we can compute stats and pairwise tests
1012 grouped = df.groupby(["problem_name", "method_name"])["fitness"]
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/frame.py:4113, in DataFrame.__getitem__(self, key)
4109
4110 if is_single_key:
4111 if self.columns.nlevels > 1:
4112 return self._getitem_multilevel(key)
-> 4113 indexer = self.columns.get_loc(key)
4114 if is_integer(indexer):
4115 indexer = [indexer]
4116 else:
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/indexes/range.py:417, in RangeIndex.get_loc(self, key)
415 raise KeyError(key) from err
416 if isinstance(key, Hashable):
--> 417 raise KeyError(key)
418 self._check_indexing_error(key)
419 raise KeyError(key)
KeyError: 'solution'
Token usage¶
plot_token_usage shows how many API tokens were consumed by each method.
[7]:
plot_token_usage(logger)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In[7], line 1
----> 1 plot_token_usage(logger)
File ~/work/BLADE/BLADE/iohblade/plots.py:1117, in plot_token_usage(logger, save, return_fig, return_df)
1114 if return_df:
1115 return token_df
1116 summary = (
-> 1117 token_df.groupby(["problem_name", "method_name"])["tokens"].sum().reset_index()
1118 )
1120 fig, ax = plt.subplots(figsize=(7, 5))
1121 sns.barplot(
1122 data=summary,
1123 x="problem_name",
(...) 1126 ax=ax,
1127 )
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/frame.py:9210, in DataFrame.groupby(self, by, axis, level, as_index, sort, group_keys, observed, dropna)
9206
9207 if level is None and by is None:
9208 raise TypeError("You have to supply one of 'by' and 'level'")
9209
-> 9210 return DataFrameGroupBy(
9211 obj=self,
9212 keys=by,
9213 axis=axis,
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/groupby/groupby.py:1331, in GroupBy.__init__(self, obj, keys, axis, level, grouper, exclusions, selection, as_index, sort, group_keys, observed, dropna)
1328 self.dropna = dropna
1330 if grouper is None:
-> 1331 grouper, exclusions, obj = get_grouper(
1332 obj,
1333 keys,
1334 axis=axis,
1335 level=level,
1336 sort=sort,
1337 observed=False if observed is lib.no_default else observed,
1338 dropna=self.dropna,
1339 )
1341 if observed is lib.no_default:
1342 if any(ping._passed_categorical for ping in grouper.groupings):
File ~/work/BLADE/BLADE/.venv/lib/python3.11/site-packages/pandas/core/groupby/grouper.py:1043, in get_grouper(obj, key, axis, level, sort, observed, validate, dropna)
1041 in_axis, level, gpr = False, gpr, None
1042 else:
-> 1043 raise KeyError(gpr)
1044 elif isinstance(gpr, Grouper) and gpr.key is not None:
1045 # Add key to exclusions
1046 exclusions.add(gpr.key)
KeyError: 'problem_name'