-
Notifications
You must be signed in to change notification settings - Fork 517
Expand file tree
/
Copy pathmigrate.py
More file actions
275 lines (239 loc) · 12.6 KB
/
migrate.py
File metadata and controls
275 lines (239 loc) · 12.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
"""Utilities for migrating to nbdev"""
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/16_migrate.ipynb.
# %% auto #0
__all__ = ['MigrateProc', 'fp_md_fm', 'migrate_nb', 'migrate_md', 'nbdev_migrate', 'nbdev_migrate_config']
# %% ../nbs/api/16_migrate.ipynb #5b687fa0-dc50-48df-8bfc-e98df34e7572
from .process import *
from .frontmatter import *
from .frontmatter import _fm2dict, _re_fm_md, _dict2fm, _insertfm
from .processors import *
from .config import get_config, read_nb, set_version, pyproject_tmpl, nbdev_defaults
from .sync import write_nb
from .showdoc import show_doc
from fastcore.all import *
import shutil
# %% ../nbs/api/16_migrate.ipynb #52f60fb5-bc54-474a-876c-9146dd092681
def _cat_slug(fmdict):
"Get the partial slug from the category front matter."
slug = '/'.join(fmdict.get('categories', ''))
return '/' + slug if slug else ''
# %% ../nbs/api/16_migrate.ipynb #9fac0338-a503-4680-984f-60153843d5ef
def _file_slug(fname):
"Get the partial slug from the filename."
p = Path(fname)
dt = '/'+p.name[:10].replace('-', '/')+'/'
return dt + p.stem[11:]
# %% ../nbs/api/16_migrate.ipynb #689bf354
def _replace_fm(d:dict, # dictionary you wish to conditionally change
k:str, # key to check
val:str,# value to check if d[k] == v
repl_dict:dict #dictionary that will be used as a replacement
):
"replace key `k` in dict `d` if d[k] == val with `repl_dict`"
if str(d.get(k, '')).lower().strip() == str(val.lower()).strip():
d.pop(k)
d = merge(d, repl_dict)
return d
def _fp_fm(d):
"create aliases for fastpages front matter to match Quarto front matter."
d = _replace_fm(d, 'search_exclude', 'true', {'search':'false'})
d = _replace_fm(d, 'hide', 'true', {'draft': 'true'})
return d
# %% ../nbs/api/16_migrate.ipynb #3146ce27
def _fp_image(d):
"Correct path of fastpages images to reference the local directory."
prefix = 'images/copied_from_nb/'
if d.get('image', '').startswith(prefix): d['image'] = d['image'].replace(prefix, '')
return d
# %% ../nbs/api/16_migrate.ipynb #244b0d01-a166-4549-9a45-1f8b1195c3c8
def _rm_quote(s):
title = re.search('''"(.*?)"''', s)
return title.group(1) if title else s
def _is_jekyll_post(path): return bool(re.search(r'^\d{4}-\d{2}-\d{2}-', Path(path).name))
def _fp_convert(fm:dict, path:Path):
"Make fastpages frontmatter Quarto complaint and add redirects."
fs = _file_slug(path)
if _is_jekyll_post(path):
fm = compose(_fp_fm, _fp_image)(fm)
if 'permalink' in fm: fm['aliases'] = [f"{fm['permalink'].strip()}"]
else: fm['aliases'] = [f'{_cat_slug(fm) + fs}']
if not fm.get('date'):
_,y,m,d,_ = fs.split('/')
fm['date'] = f'{y}-{m}-{d}'
if fm.get('summary') and not fm.get('description'): fm['description'] = fm['summary']
if fm.get('tags') and not fm.get('categories'):
if isinstance(fm['tags'], str): fm['categories'] = fm['tags'].split()
elif isinstance(fm['tags'], list): fm['categories'] = fm['tags']
for k in ['title', 'description']:
if k in fm: fm[k] = _rm_quote(fm[k])
if fm.get('comments'): fm.pop('comments') #true by itself is not a valid value for comments https://quarto.org/docs/output-formats/html-basics.html#commenting, and the default is true
return fm
# %% ../nbs/api/16_migrate.ipynb #c8440f8b-f69e-44d2-8556-8869c1eedf0f
class MigrateProc(Processor):
"Migrate fastpages front matter in notebooks to a raw cell."
def begin(self):
self.nb.frontmatter_ = _fp_convert(self.nb.frontmatter_, self.nb.path_)
if getattr(first(self.nb.cells), 'cell_type', None) == 'raw': del self.nb.cells[0]
_insertfm(self.nb, self.nb.frontmatter_)
# %% ../nbs/api/16_migrate.ipynb #d5d575f3-b3b8-487d-8740-b1ebdccf6b34
def fp_md_fm(path):
"Make fastpages front matter in markdown files quarto compliant."
p = Path(path)
md = p.read_text()
fm = _fm2dict(md, nb=False)
if fm:
fm = _fp_convert(fm, path)
return _re_fm_md.sub(_dict2fm(fm), md)
else: return md
# %% ../nbs/api/16_migrate.ipynb #1abf7dc6-4a01-4c44-bdc3-0147820091ca
_alias = merge({k:'code-fold: true' for k in ['collapse', 'collapse_input', 'collapse_hide']},
{'collapse_show':'code-fold: show', 'hide_input': 'echo: false', 'hide': 'include: false', 'hide_output': 'output: false'})
def _subv1(s): return _alias.get(s, s)
# %% ../nbs/api/16_migrate.ipynb #0dde9fe8-d3ad-47b8-bd87-85b9536e9f96
def _re_v1():
d = ['default_exp', 'export', 'exports', 'exporti', 'hide', 'hide_input', 'collapse_show', 'collapse',
'collapse_hide', 'collapse_input', 'hide_output', 'default_cls_lvl']
d += L(get_config().tst_flags).filter()
d += [s.replace('_', '-') for s in d] # allow for hyphenated version of old directives
_tmp = '|'.join(list(set(d)))
return re.compile(fr"^[ \f\v\t]*?(#)\s*({_tmp})(?!\S)", re.MULTILINE)
def _repl_directives(code_str):
def _fmt(x): return f"#| {_subv1(x[2].replace('-', '_').strip())}"
return _re_v1().sub(_fmt, code_str)
# %% ../nbs/api/16_migrate.ipynb #cddcad93-1e44-4cbe-8dbd-9538533d0f8e
def _repl_v1dir(cell):
"Replace nbdev v1 with v2 directives."
if cell.get('source') and cell.get('cell_type') == 'code':
ss = cell['source'].splitlines()
first_code = first_code_ln(ss, re_pattern=_re_v1())
if not first_code: first_code = len(ss)
if not ss: pass
else: cell['source'] = '\n'.join([_repl_directives(c) for c in ss[:first_code]] + ss[first_code:])
# %% ../nbs/api/16_migrate.ipynb #e99a0998-cbfc-46ca-a068-5695437ebc5a
_re_callout = re.compile(r'^>\s(Warning|Note|Important|Tip):(.*)', flags=re.MULTILINE)
def _co(x): return ":::{.callout-"+x[1].lower()+"}\n\n" + f"{x[2].strip()}\n\n" + ":::"
def _convert_callout(s):
"Convert nbdev v1 to v2 callouts."
return _re_callout.sub(_co, s)
# %% ../nbs/api/16_migrate.ipynb #5b19ae68-9d40-498f-bcda-38496b139a27
_re_video = re.compile(r'^>\syoutube:(.*)', flags=re.MULTILINE)
def _v(x): return "{{< " + f"video {x[1].strip()}" + " >}}"
def _convert_video(s):
"Replace nbdev v1 with v2 video embeds."
return _re_video.sub(_v, s)
# %% ../nbs/api/16_migrate.ipynb #614a8cb8-5c43-45de-b93f-8646e549cc0e
_shortcuts = compose(_convert_video, _convert_callout)
def _repl_v1shortcuts(cell):
"Replace nbdev v1 with v2 callouts."
if cell.get('source') and cell.get('cell_type') == 'markdown':
cell['source'] = _shortcuts(cell['source'])
# %% ../nbs/api/16_migrate.ipynb #9383e062-487b-4259-ab96-f427994742cc
def migrate_nb(path, overwrite=True):
"Migrate Notebooks from nbdev v1 and fastpages."
nbp = NBProcessor(path, procs=[FrontmatterProc, MigrateProc, _repl_v1shortcuts, _repl_v1dir], rm_directives=False)
nbp.process()
if overwrite: write_nb(nbp.nb, path)
return nbp.nb
# %% ../nbs/api/16_migrate.ipynb #e10f1558-95cb-49a6-b1e6-affd6fa3ecd2
def migrate_md(path, overwrite=True):
"Migrate Markdown Files from fastpages."
txt = fp_md_fm(path)
if overwrite: path.write_text(txt)
return txt
# %% ../nbs/api/16_migrate.ipynb #3eb0cd02-e1ee-4910-be82-570434b974b5
@call_parse
def nbdev_migrate(
path:str=None, # A path or glob containing notebooks and markdown files to migrate
no_skip:bool=False, # Do not skip directories beginning with an underscore
):
"Convert all markdown and notebook files in `path` from v1 to v2"
_skip_re = None if no_skip else '^[_.]'
if path is None: path = get_config().nbs_path
for f in globtastic(path, file_re='(.ipynb$)|(.md$)', skip_folder_re=_skip_re, func=Path):
try:
if f.name.endswith('.ipynb'): migrate_nb(f)
if f.name.endswith('.md'): migrate_md(f)
except Exception as e: raise Exception(f'Error in migrating file: {f}') from e
# %% ../nbs/api/16_migrate.ipynb #ad76e503
_license_map = {'apache2': 'Apache-2.0', 'mit': 'MIT', 'gpl2': 'GPL-2.0', 'gpl3': 'GPL-3.0', 'bsd3': 'BSD-3-Clause'}
# %% ../nbs/api/16_migrate.ipynb #124b1c9b
def _migrate_workflows(path):
"Update GitHub workflow files to use nbdev3 workflows"
wf_path = Path(path) / '.github/workflows'
if not wf_path.exists(): return
replacements = [
('fastai/workflows/quarto-ghp@', 'fastai/workflows/quarto-ghp3@'),
('fastai/workflows/nbdev-ci@', 'fastai/workflows/nbdev3-ci@'),
]
for f in (*wf_path.glob('*.yml'), *wf_path.glob('*.yaml')):
txt = f.read_text()
for old, new in replacements: txt = txt.replace(old, new)
f.write_text(txt)
# %% ../nbs/api/16_migrate.ipynb #1ca2d1b3
def _toml_val(v):
if v.lower() in ('true','false'): return v.lower()
return repr(v)
def _py_val(v):
try: return str2bool(v)
except: return v
def _fmt_script(s):
name,val = s.strip().split('=')
return f'{name.strip()} = "{val.strip()}"'
def _build_classifiers(d):
"Build list of classifier strings from config dict"
_status_map = {'1': 'Planning', '2': 'Pre-Alpha', '3': 'Alpha', '4': 'Beta', '5': 'Production/Stable', '6': 'Mature', '7': 'Inactive'}
cs = ['Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only']
status,audience,language = d.get('status', '').strip(), d.get('audience', '').strip(), d.get('language', '').strip()
if status and status in _status_map: cs.insert(0, f"Development Status :: {status} - {_status_map[status]}")
if audience: cs.insert(0, f"Intended Audience :: {audience}")
if language: cs.insert(0, f"Natural Language :: {language}")
return cs
# %% ../nbs/api/16_migrate.ipynb #796a2c4b
def _nbdev_migrate_config(d, path): # Config dict from settings.ini
"Migrate settings.ini dict to pyproject.toml string"
repo = d.get('repo', '')
user = d.get('user', '')
lib_path = d.get('lib_path', repo.replace('-', '_'))
branch = d.get('branch', 'main')
git_url = d.get('git_url') or (f"https://github.com/{user}/{repo}" if user else '')
doc_host = d.get('doc_host', '')
doc_baseurl = d.get('doc_baseurl', '')
doc_url = (doc_host.rstrip('/') + doc_baseurl) if doc_host else (f"https://{user}.github.io/{repo}/" if user else '')
set_version(path/lib_path, d.get('version', '0.0.1'))
lib_name = d.get('lib_name', repo)
txt = pyproject_tmpl.format(name=lib_name, lib_path=lib_path, description=d.get('description', ''),
min_python=d.get('min_python', '3.10'), license=_license_map.get(d.get('license', ''), d.get('license', 'Apache-2.0')),
author=d.get('author', ''), author_email=d.get('author_email', ''),
keywords=d.get('keywords', 'nbdev').split(), git_url=git_url, doc_url=doc_url, branch=branch)
# Add dependencies (combine requirements + pip_requirements)
reqs = d.get('requirements', '').split() + d.get('pip_requirements', '').split()
if reqs: txt = txt.replace('dependencies = []', f'dependencies = {reqs}')
dev_reqs = d.get('dev_requirements', '').split()
if dev_reqs: txt = txt.replace('[tool.setuptools', f'[project.optional-dependencies]\ndev = {dev_reqs}\n\n[tool.setuptools', 1)
# Add console_scripts
scripts = d.get('console_scripts', '').strip()
if scripts:
scripts_lines = [_fmt_script(s) for s in scripts.split('\n') if s.strip()]
scripts_toml = '\n[project.scripts]\n' + '\n'.join(scripts_lines)
txt = txt.replace('[tool.setuptools', scripts_toml + '\n\n[tool.setuptools', 1)
_classifiers_default = 'classifiers = [\n "Programming Language :: Python :: 3",\n "Programming Language :: Python :: 3 :: Only",\n]'
txt = txt.replace(_classifiers_default, 'classifiers = ' + repr(_build_classifiers(d)).replace("'", '"'))
nbdev_settings = {k:d[k] for k in ('nbs_path','doc_path','branch','recursive','readme_nb','tst_flags',
'clean_ids','clear_all','put_version_in_init','jupyter_hooks','custom_sidebar','title')
if k in d and _py_val(d[k]) != nbdev_defaults.get(k) and not (k=='title' and d[k]==repo)}
if nbdev_settings:
nbdev_toml = '\n'.join(f'{k} = {_toml_val(v)}' for k,v in nbdev_settings.items())
txt = txt.rstrip() + '\n' + nbdev_toml + '\n'
_migrate_workflows(path)
return txt
# %% ../nbs/api/16_migrate.ipynb #a9534478
@call_parse
def nbdev_migrate_config(path:str='.'): # Project root containing settings.ini
"Migrate settings.ini to pyproject.toml"
path = Path(path)
ini = path/'settings.ini'
if not ini.exists(): return print(f"No settings.ini found at {ini}")
cfg = Config(path, 'settings.ini')
txt = _nbdev_migrate_config(cfg.d, path)
(path/'pyproject.toml').write_text(txt)
print(f"Created {path/'pyproject.toml'}. You can now delete {ini} and setup.py (if present)")