I had too many failures for one post, so this post describes even more ways not to evaluate models with FastAI.

import ast
import dill
from fastai.vision.all import *
learner = load_learner(Path(os.getenv('MODELS')) / 'siamese_catsvdogs.pkl', cpu=False, pickle_module=dill)
df = pd.read_csv('siamese_data.csv', index_col=0)
df.head()
items label is_valid
0 ['/home/julius/.fastai/data/oxford-iiit-pet/images/keeshond_13.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/leonberger_184.jpg'] False False
1 ['/home/julius/.fastai/data/oxford-iiit-pet/images/Russian_Blue_175.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/Russian_Blue_24.jpg'] True False
2 ['/home/julius/.fastai/data/oxford-iiit-pet/images/german_shorthaired_145.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/german_shorthaired_147.jpg'] True False
3 ['/home/julius/.fastai/data/oxford-iiit-pet/images/beagle_17.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/american_bulldog_110.jpg'] False False
4 ['/home/julius/.fastai/data/oxford-iiit-pet/images/leonberger_11.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/Persian_5.jpg'] False False
df.iloc[0]
items       ['/home/julius/.fastai/data/oxford-iiit-pet/images/keeshond_13.jpg', '/home/julius/.fastai/data/oxford-iiit-pet/images/leonberger_184.jpg']
label                                                                                                                                             False
is_valid                                                                                                                                          False
Name: 0, dtype: object

In theory you should be able to do learner.predict(pair_df.iloc[0]). This works for simple networks but fails in this case. Instead, you can just do the first two lines of that function:

dl = self.dls.test_dl([item], rm_type_tfms=rm_type_tfms, num_workers=0)
inp, preds, _, dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
dl = learner.dls.test_dl(df)
try:
    learner.get_preds(dl=dl, with_input=True)
except (IsADirectoryError, FileNotFoundError) as err:
    print(err)
0.00% [0/119 00:00<?]
Caught FileNotFoundError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 39, in fetch
    data = next(self.dataset_iter)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 140, in create_batches
    yield from map(self.do_batch, self.chunkify(res))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/basics.py", line 230, in chunked
    res = list(itertools.islice(it, chunk_sz))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 155, in do_item
    try: return self.after_item(self.create_item(s))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 162, in create_item
    if self.indexed: return self.dataset[s or 0]
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in __getitem__
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in <listcomp>
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 509, in __getitem__
    return self._after_item(res) if is_indexer(idx) else res.map(self._after_item)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 455, in _after_item
    return self.tfms(o)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 299, in __call__
    return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 234, in compose_tfms
    x = f(x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 117, in __call__
    return self._call("encodes", x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 132, in _call
    return self._do_call(getattr(self, fn), x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 139, in _do_call
    return retain_type(f(x, **kwargs), x, ret)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/dispatch.py", line 161, in __call__
    return f(*args, **kwargs)
  File "/tmp/ipykernel_15574/1954726133.py", line 235, in create
  File "/tmp/ipykernel_15574/1954726133.py", line -1, in <genexpr>
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 185, in create
    return cls(load_image(fn, **merge(cls._open_args, kwargs)))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 151, in load_image
    im = Image.open(fn)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/PIL/Image.py", line 3046, in open
    fp = builtins.open(filename, "rb")
FileNotFoundError: [Errno 2] No such file or directory: '['

Converting to a tuple also doesn’t help.

df['items'] = df['items'].apply(ast.literal_eval)
dl = learner.dls.test_dl(df)
try:
    learner.get_preds(dl=dl, with_input=True)
except (IsADirectoryError, FileNotFoundError, AttributeError) as err:
    print(err)
0.00% [0/119 00:00<?]
Caught AttributeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/PIL/Image.py", line 3050, in open
    fp.seek(0)
AttributeError: 'list_iterator' object has no attribute 'seek'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 39, in fetch
    data = next(self.dataset_iter)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 140, in create_batches
    yield from map(self.do_batch, self.chunkify(res))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/basics.py", line 230, in chunked
    res = list(itertools.islice(it, chunk_sz))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 155, in do_item
    try: return self.after_item(self.create_item(s))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 162, in create_item
    if self.indexed: return self.dataset[s or 0]
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in __getitem__
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in <listcomp>
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 509, in __getitem__
    return self._after_item(res) if is_indexer(idx) else res.map(self._after_item)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 455, in _after_item
    return self.tfms(o)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 299, in __call__
    return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 234, in compose_tfms
    x = f(x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 117, in __call__
    return self._call("encodes", x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 132, in _call
    return self._do_call(getattr(self, fn), x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 139, in _do_call
    return retain_type(f(x, **kwargs), x, ret)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/dispatch.py", line 161, in __call__
    return f(*args, **kwargs)
  File "/tmp/ipykernel_15574/1954726133.py", line 235, in create
  File "/tmp/ipykernel_15574/1954726133.py", line -1, in <genexpr>
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 185, in create
    return cls(load_image(fn, **merge(cls._open_args, kwargs)))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 151, in load_image
    im = Image.open(fn)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/PIL/Image.py", line 3052, in open
    fp = io.BytesIO(fp.read())
AttributeError: 'list_iterator' object has no attribute 'read'
df['items'] = df['items'].apply(lambda x: tuple(x))
df.head()
items label is_valid
0 (/home/julius/.fastai/data/oxford-iiit-pet/images/keeshond_13.jpg, /home/julius/.fastai/data/oxford-iiit-pet/images/leonberger_184.jpg) False False
1 (/home/julius/.fastai/data/oxford-iiit-pet/images/Russian_Blue_175.jpg, /home/julius/.fastai/data/oxford-iiit-pet/images/Russian_Blue_24.jpg) True False
2 (/home/julius/.fastai/data/oxford-iiit-pet/images/german_shorthaired_145.jpg, /home/julius/.fastai/data/oxford-iiit-pet/images/german_shorthaired_147.jpg) True False
3 (/home/julius/.fastai/data/oxford-iiit-pet/images/beagle_17.jpg, /home/julius/.fastai/data/oxford-iiit-pet/images/american_bulldog_110.jpg) False False
4 (/home/julius/.fastai/data/oxford-iiit-pet/images/leonberger_11.jpg, /home/julius/.fastai/data/oxford-iiit-pet/images/Persian_5.jpg) False False
dl = learner.dls.test_dl(df)
try:
    learner.get_preds(dl=dl, with_input=True)
except (IsADirectoryError, FileNotFoundError, AttributeError) as err:
    print(err)
0.00% [0/119 00:00<?]
Caught IsADirectoryError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 39, in fetch
    data = next(self.dataset_iter)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 140, in create_batches
    yield from map(self.do_batch, self.chunkify(res))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/basics.py", line 230, in chunked
    res = list(itertools.islice(it, chunk_sz))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 155, in do_item
    try: return self.after_item(self.create_item(s))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/load.py", line 162, in create_item
    if self.indexed: return self.dataset[s or 0]
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in __getitem__
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 557, in <listcomp>
    res = tuple([tl[it] for tl in self.tls])
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 509, in __getitem__
    return self._after_item(res) if is_indexer(idx) else res.map(self._after_item)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/data/core.py", line 455, in _after_item
    return self.tfms(o)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 299, in __call__
    return compose_tfms(o, tfms=self.fs, split_idx=self.split_idx)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 234, in compose_tfms
    x = f(x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 117, in __call__
    return self._call("encodes", x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 132, in _call
    return self._do_call(getattr(self, fn), x, **kwargs)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 141, in _do_call
    res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 141, in <genexpr>
    res = tuple(self._do_call(f, x_, **kwargs) for x_ in x)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/transform.py", line 139, in _do_call
    return retain_type(f(x, **kwargs), x, ret)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastcore/dispatch.py", line 161, in __call__
    return f(*args, **kwargs)
  File "/tmp/ipykernel_15574/1954726133.py", line 235, in create
  File "/tmp/ipykernel_15574/1954726133.py", line -1, in <genexpr>
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 185, in create
    return cls(load_image(fn, **merge(cls._open_args, kwargs)))
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/fastai/vision/core.py", line 151, in load_image
    im = Image.open(fn)
  File "/home/julius/miniconda3/envs/pt2/lib/python3.10/site-packages/PIL/Image.py", line 3046, in open
    fp = builtins.open(filename, "rb")
IsADirectoryError: [Errno 21] Is a directory: '/'