Coverage for dlabel/traefik.py: 88%
289 statements
« prev ^ index » next coverage.py v7.10.0, created at 2025-07-25 23:32 +0000
1import docker
2import re
3import io
4import yaml
5import toml
6from pathlib import Path
7from logging import getLogger
8from .traefik_conf import TraefikConfig, HttpMiddleware, HttpService, ProviderConfig
9from .util import download_files
11_log = getLogger(__name__)
14def find_block(conf: list[dict], directive: str):
15 for c in conf: 15 ↛ exitline 15 didn't return from function 'find_block' because the loop on line 15 didn't complete
16 if c.get("directive") == directive:
17 _log.debug("found directive %s: %s", directive, c)
18 yield c
21def find_server_block(conf: dict, server_name: str) -> list | None:
22 for conf in conf.get("config", []): 22 ↛ 28line 22 didn't jump to line 28 because the loop on line 22 didn't complete
23 for http in find_block(conf.get("parsed", []), "http"): 23 ↛ 22line 23 didn't jump to line 22 because the loop on line 23 didn't complete
24 for srv in find_block(http.get("block", []), "server"): 24 ↛ 23line 24 didn't jump to line 23 because the loop on line 24 didn't complete
25 for name in find_block(srv.get("block", []), "server_name"): 25 ↛ 24line 25 didn't jump to line 24 because the loop on line 25 didn't complete
26 if server_name in name.get("args", []): 26 ↛ 25line 26 didn't jump to line 25 because the condition on line 26 was always true
27 return srv.get("block", [])
28 return None
31def middleware_compress(mdl: HttpMiddleware) -> list[dict]:
32 res = []
33 if mdl.compress:
34 res.append({
35 "directive": "gzip",
36 "args": ["on"],
37 })
38 if not isinstance(mdl.compress, bool): 38 ↛ 49line 38 didn't jump to line 49 because the condition on line 38 was always true
39 if mdl.compress.includedcontenttypes: 39 ↛ 44line 39 didn't jump to line 44 because the condition on line 39 was always true
40 res.append({
41 "directive": "gzip_types",
42 "args": mdl.compress.includedcontenttypes,
43 })
44 if mdl.compress.minresponsebodybytes: 44 ↛ 49line 44 didn't jump to line 49 because the condition on line 44 was always true
45 res.append({
46 "directive": "gzip_min_length",
47 "args": [str(mdl.compress.minresponsebodybytes)],
48 })
49 return res
52def middleware_compress_apache(mdl: HttpMiddleware) -> list[str]:
53 if mdl.compress:
54 if not isinstance(mdl.compress, bool) and mdl.compress.includedcontenttypes: 54 ↛ 56line 54 didn't jump to line 56 because the condition on line 54 was always true
55 return [f"AddOutputFilterByType DEFLATE {' '.join(mdl.compress.includedcontenttypes)}"]
56 return ["SetOutputFilter DEFLATE"]
57 return []
60def middleware_headers(mdl: HttpMiddleware) -> list[dict]:
61 res = []
62 if mdl.headers:
63 if mdl.headers.customrequestheaders: 63 ↛ 69line 63 didn't jump to line 69 because the condition on line 63 was always true
64 for k, v in mdl.headers.customrequestheaders.items():
65 res.append({
66 "directive": "proxy_set_header",
67 "args": [k, v],
68 })
69 if mdl.headers.customresponseheaders: 69 ↛ 75line 69 didn't jump to line 75 because the condition on line 69 was always true
70 for k, v in mdl.headers.customresponseheaders.items():
71 res.append({
72 "directive": "add_header",
73 "args": [k, v],
74 })
75 return res
78def middleware_headers_apache(mdl: HttpMiddleware) -> list[str]:
79 res = []
80 if mdl.headers:
81 if mdl.headers.customrequestheaders: 81 ↛ 84line 81 didn't jump to line 84 because the condition on line 81 was always true
82 for k, v in mdl.headers.customrequestheaders.items():
83 res.append(f"RequestHeader append {k} {v}")
84 if mdl.headers.customresponseheaders: 84 ↛ 87line 84 didn't jump to line 87 because the condition on line 84 was always true
85 for k, v in mdl.headers.customresponseheaders.items():
86 res.append(f"Header append {k} {v}")
87 return res
90def middleware2nginx(mdlconf: list[HttpMiddleware]) -> list[dict]:
91 _log.debug("apply middleware: %s", mdlconf)
92 res = []
93 del_prefix = []
94 add_prefix = "/"
95 for mdl in mdlconf:
96 res.extend(middleware_compress(mdl))
97 res.extend(middleware_headers(mdl))
98 if mdl.stripprefix and mdl.stripprefix.prefixes:
99 del_prefix.extend([re.escape(x) for x in mdl.stripprefix.prefixes])
100 if mdl.stripprefixregex and mdl.stripprefixregex.regex: 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true
101 del_prefix.extend(mdl.stripprefixregex.regex)
102 if mdl.addprefix and mdl.addprefix.prefix: 102 ↛ 103line 102 didn't jump to line 103 because the condition on line 102 was never true
103 add_prefix = mdl.addprefix.prefix
104 if del_prefix or add_prefix != "/":
105 res.append({
106 "directive": "rewrite",
107 "args": [f"{'|'.join(del_prefix)}(.*)", f"{add_prefix}$1", "break"],
108 })
109 _log.debug("middleware2nginx result: %s -> %s", mdlconf, res)
110 return res
113def rule2locationkey(rule: str) -> list[str]:
114 m = re.match(r"^PathPrefix\(`(?P<prefix>[^`]+)`\)$", rule)
115 location_key = []
116 if m:
117 location_key = [m.group("prefix")]
118 else:
119 m = re.match(r"^Path\(`(?P<path>[^`]+)`\)$", rule)
120 if m: 120 ↛ 122line 120 didn't jump to line 122 because the condition on line 120 was always true
121 location_key = ["=", m.group('path')]
122 return location_key
125def traefik_label_config(labels: dict[str, str], host: str | None, ipaddr: str | None):
126 res = TraefikConfig()
127 for k, v in labels.items():
128 if k == "traefik.enable":
129 continue
130 if k.startswith("traefik."):
131 _, k1 = k.split(".", 1)
132 m = re.match(r"http\.services\.([^\.]+)\.loadbalancer\.server\.port", k1)
133 if m:
134 res = res.setbyaddr(["http", "services", m.group(1), "loadbalancer", "server", "host"], host)
135 res = res.setbyaddr(["http", "services", m.group(1), "loadbalancer", "server", "ipaddress"], ipaddr)
136 res = res.setbyaddr(k1.split("."), int(v))
137 else:
138 res = res.setbyaddr(k1.split("."), v)
139 return res
142def traefik_container_config(ctn: docker.models.containers.Container):
143 from_args = TraefikConfig()
144 from_envs = TraefikConfig()
145 from_conf = TraefikConfig()
146 for arg in ctn.attrs.get("Args", []):
147 if arg.startswith("--") and "=" in arg: 147 ↛ 146line 147 didn't jump to line 146 because the condition on line 147 was always true
148 k, v = arg.split("=", 1)
149 from_args = from_args.setbyaddr(k[2:].split("."), v)
150 for env in ctn.attrs.get("Config", {}).get("Env", []):
151 if env.startswith("TRAEFIK_") and "=" in env:
152 k, v = env[8:].split("=", 1)
153 from_envs = from_envs.setbyaddr(k.split("_"), v)
154 provider = ProviderConfig()
155 provider = provider.merge(from_args.providers)
156 provider = provider.merge(from_envs.providers)
157 _log.debug("provider config: %s (arg=%s, env=%s)", provider, from_args, from_envs)
158 if provider.file:
159 _log.debug("loading file: %s", provider.file)
160 to_load = provider.file.filename or provider.file.directory
161 if to_load: 161 ↛ 173line 161 didn't jump to line 173 because the condition on line 161 was always true
162 for _, tinfo, bin in download_files(ctn, to_load):
163 _log.debug("fn=%s, bin(len)=%s", tinfo.name, len(bin))
164 if tinfo.name.endswith(".yml") or tinfo.name.endswith(".yaml"): 164 ↛ 166line 164 didn't jump to line 166 because the condition on line 164 was always true
165 loaded = yaml.safe_load(bin)
166 elif tinfo.name.endswith(".toml"):
167 loaded = toml.loads(bin)
168 else:
169 _log.info("unknown format: %s", tinfo.name)
170 continue
171 _log.debug("load(dict): %s", loaded)
172 from_conf = from_conf.merge(TraefikConfig.model_validate(loaded))
173 return from_args, from_envs, from_conf
176def traefik_dump(client: docker.DockerClient) -> TraefikConfig:
177 """extract traefik configuration"""
178 from_conf = TraefikConfig()
179 from_args = TraefikConfig()
180 from_envs = TraefikConfig()
181 from_label = TraefikConfig()
182 for ctn in client.containers.list():
183 if ctn.status != "running": 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 _log.debug("skip %s (not running: %s)", ctn.name, ctn.status)
185 continue
186 if "traefik" in ctn.image.tags[0]:
187 _log.debug("traefik container: %s", ctn.name)
188 from_args, from_envs, from_conf = traefik_container_config(ctn)
189 _log.debug("loaded: args=%s, conf=%s", from_args, from_conf)
190 if ctn.labels.get("traefik.enable") in ("true",):
191 _log.debug("traefik enabled container: %s", ctn.name)
192 host = ctn.name
193 addrs = [x["IPAddress"] for x in ctn.attrs["NetworkSettings"]["Networks"].values()]
194 if len(addrs) != 0:
195 addr = addrs[0]
196 else:
197 addr = ""
198 ctn_label = traefik_label_config(ctn.labels, host, addr)
199 from_label = from_label.merge(ctn_label)
200 _log.debug("conf: %s", from_conf)
201 _log.debug("arg: %s", from_args)
202 _log.debug("label: %s", from_label)
203 res = from_conf.merge(from_envs)
204 res = res.merge(from_args)
205 res = res.merge(from_label)
206 return res
209def get_backend(svc: HttpService, ipaddr: bool = False) -> list[str]:
210 if svc.loadbalancer is None: 210 ↛ 211line 210 didn't jump to line 211 because the condition on line 210 was never true
211 return []
212 backend_urls = []
213 if svc.loadbalancer.servers:
214 backend_urls.extend([x.url.removeprefix("http://") for x in svc.loadbalancer.servers if x.url])
215 if svc.loadbalancer.server and svc.loadbalancer.server.port:
216 if ipaddr:
217 backend_urls.append(f"{svc.loadbalancer.server.ipaddress}:{svc.loadbalancer.server.port}")
218 else:
219 backend_urls.append(f"{svc.loadbalancer.server.host}:{svc.loadbalancer.server.port}")
220 return backend_urls
223def traefik2nginx(traefik_file: TraefikConfig | str, output: io.IOBase, baseconf: str | None,
224 server_url: str, ipaddr: bool):
225 """generate nginx configuration from traefik configuration"""
226 import crossplane
227 import urllib.parse
228 ps = urllib.parse.urlparse(server_url, scheme="http", allow_fragments=False)
229 if baseconf: 229 ↛ 230line 229 didn't jump to line 230 because the condition on line 229 was never true
230 nginx_confs = crossplane.parse(baseconf)
231 else:
232 import tempfile
233 minconf = """
234user nginx;
235worker_processes auto;
236error_log /dev/stderr notice;
237events {worker_connections 512;}
238http {server {listen %s default_server; server_name %s;}}
239""" % (ps.port or 80, ps.hostname)
240 with tempfile.NamedTemporaryFile("r+") as tf:
241 tf.write(minconf)
242 tf.seek(0)
243 nginx_confs = crossplane.parse(tf.name, combine=True)
244 target = find_server_block(nginx_confs, ps.hostname or "localhost")
245 _log.debug("target: %s", target)
246 assert target is not None
247 if isinstance(traefik_file, TraefikConfig):
248 traefik_config = traefik_file
249 else:
250 traefik_config = TraefikConfig.model_validate(yaml.safe_load(traefik_file))
251 if not traefik_config.http: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true
252 raise Exception(f"http not defined: {traefik_config}")
253 services = traefik_config.http.services or {}
254 routers = traefik_config.http.routers or {}
255 middlewares = traefik_config.http.middlewares or {}
256 _log.debug("all middlewares: %s", middlewares)
257 for location in set(services.keys()) & set(routers.keys()):
258 route, svc = routers[location], services[location]
259 rule = route.rule or ""
260 middleware_names = route.middlewares or []
261 _log.debug("middleware_names: %s", middleware_names)
262 location_keys = [rule2locationkey(x) for x in rule.split("||")]
263 middles: list[HttpMiddleware] = [i for i in [middlewares.get(
264 x.split("@", 1)[0]) for x in middleware_names] if i is not None]
265 _log.debug("middles: %s", middles)
266 backend_urls = get_backend(svc, ipaddr)
267 target.append({
268 "directive": "#",
269 "comment": f" {location}: {', '.join([' '.join(x) for x in location_keys])} -> {', '.join(backend_urls)}",
270 "line": 1
271 })
272 if len(backend_urls) > 1:
273 _log.info("multiple backend urls: %s", backend_urls)
274 target.append({
275 "directive": "upstream",
276 "args": [location],
277 "block": [{"directive": "server", "args": [x]} for x in backend_urls],
278 })
279 backend = location
280 else:
281 backend = backend_urls[0]
282 blk = [{"directive": "proxy_pass", "args": [f"http://{backend}"]}]
283 blk.extend(middleware2nginx(middles))
284 for lk in location_keys:
285 target.append({
286 "directive": "location",
287 "args": lk,
288 "block": blk,
289 })
290 for conf in nginx_confs.get("config", []):
291 output.write(crossplane.build(conf.get("parsed", [])))
292 output.write("\n")
295def apache_insert2vf(base_conf: list[str], location_conf: list[str]) -> list[str]:
296 if "</VirtualHost>" in base_conf: 296 ↛ 300line 296 didn't jump to line 300 because the condition on line 296 was always true
297 insert_to = base_conf.index("</VirtualHost>")
298 indent = len(base_conf[insert_to - 1]) - len(base_conf[insert_to - 1].lstrip())
299 else:
300 insert_to = len(base_conf)
301 indent = 0
302 _log.debug("insert to %s", insert_to)
303 return base_conf[:insert_to] + [""] + [" " * indent + x for x in location_conf] + [""] + base_conf[insert_to:]
306def middleware2apache(mdlconf: list[HttpMiddleware]) -> list[str]:
307 _log.debug("apply middleware: %s", mdlconf)
308 res = []
309 del_prefix = []
310 add_prefix = "/"
311 for mdl in mdlconf:
312 res.extend(middleware_compress_apache(mdl))
313 res.extend(middleware_headers_apache(mdl))
314 if mdl.stripprefix and mdl.stripprefix.prefixes:
315 del_prefix.extend([re.escape(x) for x in mdl.stripprefix.prefixes])
316 if mdl.stripprefixregex and mdl.stripprefixregex.regex: 316 ↛ 317line 316 didn't jump to line 317 because the condition on line 316 was never true
317 del_prefix.extend(mdl.stripprefixregex.regex)
318 if mdl.addprefix and mdl.addprefix.prefix: 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true
319 add_prefix = mdl.addprefix.prefix
320 if del_prefix or add_prefix != "/":
321 res.append("RewriteEngine On")
322 res.append(f"RewriteRule {'|'.join(del_prefix)}(.*) {add_prefix}$1")
323 _log.debug("middleware2apache result: %s -> %s", mdlconf, res)
324 return res
327def traefik2apache(traefik_file: TraefikConfig | str, output: io.IOBase, baseconf: str | None,
328 server_url: str, ipaddr: bool):
329 """generate apache virtualhost configuration from traefik configuration"""
330 if baseconf: 330 ↛ 331line 330 didn't jump to line 331 because the condition on line 330 was never true
331 apconf = Path(baseconf).read_text()
332 else:
333 import urllib.parse
334 ps = urllib.parse.urlparse(server_url, scheme="http", allow_fragments=False)
335 apconf = """
336<VirtualHost *:%s>
337 ServerName %s
338 ErrorLog /dev/stderr
339</VirtualHost>
340""" % (ps.port or 80, ps.hostname)
342 if isinstance(traefik_file, TraefikConfig): 342 ↛ 343line 342 didn't jump to line 343 because the condition on line 342 was never true
343 traefik_config = traefik_file
344 else:
345 traefik_config = TraefikConfig.model_validate(yaml.safe_load(traefik_file))
346 if not traefik_config.http: 346 ↛ 347line 346 didn't jump to line 347 because the condition on line 346 was never true
347 raise Exception(f"http not defined: {traefik_config}")
348 services = traefik_config.http.services or {}
349 routers = traefik_config.http.routers or {}
350 middlewares = traefik_config.http.middlewares or {}
351 _log.debug("all middlewares: %s", middlewares)
352 res = []
353 for location in set(services.keys()) & set(routers.keys()):
354 route, svc = routers[location], services[location]
355 rule = route.rule or ""
356 _log.debug("rules: %s", rule)
357 middleware_names = route.middlewares or []
358 _log.debug("middleware_names: %s", middleware_names)
359 location_keys = [rule2locationkey(x) for x in rule.split("||")]
360 _log.debug("location: %s", location_keys)
361 backend_urls = get_backend(svc, ipaddr)
362 if len(backend_urls) == 1:
363 backend_to = f"http://{backend_urls[0]}"
364 else:
365 res.append(f"<Proxy balancer://{location}>")
366 for b in backend_urls:
367 res.append(f" BalancerMember http://{b}")
368 res.append("</Proxy>")
369 backend_to = f"balancer://{location}"
370 middles: list[HttpMiddleware] = [i for i in [middlewares.get(
371 x.split("@", 1)[0]) for x in middleware_names] if i is not None]
372 _log.debug("middles: %s", middles)
373 mdlconf = middleware2apache(middles)
374 for loc in location_keys:
375 if len(loc) == 1:
376 res.append(f"<Location {loc[0]}>")
377 elif loc[0] == "=": 377 ↛ 379line 377 didn't jump to line 379 because the condition on line 377 was always true
378 res.append(f"<Location ~ \"^{re.escape(loc[1])}$\">")
379 res.append(f" ProxyPass {backend_to}")
380 res.append(f" ProxyPassReverse {backend_to}")
381 res.extend([f" {i}" for i in mdlconf])
382 res.append("</Location>")
383 print("\n".join(apache_insert2vf(apconf.splitlines(), res)), file=output)