From bc02c123e63abca4c5006ab50444344f8067374b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jan 2020 15:32:09 -0800 Subject: [PATCH] Monitor backups with PagerDuty hook integration (#245). --- NEWS | 2 + README.md | 1 + borgmatic/config/schema.yaml | 9 ++++ borgmatic/hooks/dispatch.py | 3 +- borgmatic/hooks/monitor.py | 2 +- borgmatic/hooks/pagerduty.py | 62 ++++++++++++++++++++++++++++ docs/how-to/monitor-your-backups.md | 41 ++++++++++++++---- docs/static/pagerduty.png | Bin 0 -> 20107 bytes tests/unit/hooks/test_pagerduty.py | 35 ++++++++++++++++ 9 files changed, 146 insertions(+), 9 deletions(-) create mode 100644 borgmatic/hooks/pagerduty.py create mode 100644 docs/static/pagerduty.png create mode 100644 tests/unit/hooks/test_pagerduty.py diff --git a/NEWS b/NEWS index 151e0d608..1be3d09cd 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.5.0 + * #245: Monitor backups with PagerDuty hook integration. See the documentation for more + information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook * #255: Add per-action hooks: "before_prune", "after_prune", "before_check", and "after_check". * #274: Add ~/.config/borgmatic.d as another configuration directory default. * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag. diff --git a/README.md b/README.md index 16dc0eaa8..b2ba6e0a8 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Healthchecks      Cronitor      Cronhub      +PagerDuty      rsync.net      BorgBase      diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3a09fdf57..a228e7a31 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -567,6 +567,15 @@ map: for details. example: https://cronitor.link/d3x0c1 + pagerduty: + type: str + desc: | + PagerDuty integration key used to notify PagerDuty when a backup errors. Create + an account at https://www.pagerduty.com/ if you'd like to use this service. See + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook + for details. + example: + a177cad45bd374409f78906a810a3074 cronhub: type: str desc: | diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 206b0d1c3..6c05cad9c 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -1,6 +1,6 @@ import logging -from borgmatic.hooks import cronhub, cronitor, healthchecks, mysql, postgresql +from borgmatic.hooks import cronhub, cronitor, healthchecks, mysql, pagerduty, postgresql logger = logging.getLogger(__name__) @@ -8,6 +8,7 @@ HOOK_NAME_TO_MODULE = { 'healthchecks': healthchecks, 'cronitor': cronitor, 'cronhub': cronhub, + 'pagerduty': pagerduty, 'postgresql_databases': postgresql, 'mysql_databases': mysql, } diff --git a/borgmatic/hooks/monitor.py b/borgmatic/hooks/monitor.py index aee2b8f56..c4cf576e1 100644 --- a/borgmatic/hooks/monitor.py +++ b/borgmatic/hooks/monitor.py @@ -1,6 +1,6 @@ from enum import Enum -MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub') +MONITOR_HOOK_NAMES = ('healthchecks', 'cronitor', 'cronhub', 'pagerduty') class State(Enum): diff --git a/borgmatic/hooks/pagerduty.py b/borgmatic/hooks/pagerduty.py new file mode 100644 index 000000000..0e613cc56 --- /dev/null +++ b/borgmatic/hooks/pagerduty.py @@ -0,0 +1,62 @@ +import datetime +import json +import logging +import platform + +import requests + +from borgmatic.hooks import monitor + +logger = logging.getLogger(__name__) + +EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue' + + +def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run): + ''' + If this is an error state, create a PagerDuty event with the given integration key. Use the + given configuration filename in any log entries. If this is a dry run, then don't actually + create an event. + ''' + if state != monitor.State.FAIL: + logger.debug( + '{}: Ignoring unsupported monitoring {} in PagerDuty hook'.format( + config_filename, state.name.lower() + ) + ) + return + + dry_run_label = ' (dry run; not actually sending)' if dry_run else '' + logger.info('{}: Sending failure event to PagerDuty {}'.format(config_filename, dry_run_label)) + + if dry_run: + return + + hostname = platform.node() + local_timestamp = ( + datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).astimezone().isoformat() + ) + payload = json.dumps( + { + 'routing_key': integration_key, + 'event_action': 'trigger', + 'payload': { + 'summary': 'backup failed on {}'.format(hostname), + 'severity': 'error', + 'source': hostname, + 'timestamp': local_timestamp, + 'component': 'borgmatic', + 'group': 'backups', + 'class': 'backup failure', + 'custom_details': { + 'hostname': hostname, + 'configuration filename': config_filename, + 'server time': local_timestamp, + }, + }, + } + ) + logger.debug('{}: Using PagerDuty payload: {}'.format(config_filename, payload)) + + logging.getLogger('urllib3').setLevel(logging.ERROR) + requests.post(EVENTS_API_URL, data=payload.encode('utf-8')) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index c56a15136..064c4080e 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -28,14 +28,15 @@ hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hoo below for how to configure this. 4. **borgmatic monitoring hooks**: This feature integrates with monitoring services like [Healthchecks](https://healthchecks.io/), -[Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings -these services whenever borgmatic runs. That way, you'll receive an alert when -something goes wrong or the service doesn't hear from borgmatic for a -configured interval. See -[Healthchecks +[Cronitor](https://cronitor.io), [Cronhub](https://cronhub.io), and +[PagerDuty](https://www.pagerduty.com/) and pings these services whenever +borgmatic runs. That way, you'll receive an alert when something goes wrong or +(for certain hooks) the service doesn't hear from borgmatic for a configured +interval. See [Healthchecks hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor -hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub -hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), [Cronhub +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook), and +[PagerDuty hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) below for how to configure this. 3. **Third-party monitoring software**: You can use traditional monitoring software to consume borgmatic JSON output and track when the last @@ -200,6 +201,32 @@ mechanisms](https://docs.cronhub.io/integrations.html) when backups fail or it doesn't hear from borgmatic for a certain period of time. +## PagerDuty hook + +[PagerDuty](https://cronhub.io/) provides incident monitoring and alerting, +and borgmatic has built-in integration with it. Once you create a PagerDuty +account and service +on their site, all you need to do is configure borgmatic with the unique +"Integration Key" for your service. Here's an example: + + +```yaml +hooks: + pagerduty: a177cad45bd374409f78906a810a3074 +``` + +With this hook in place, borgmatic creates a PagerDuty event for your service +whenever backups fail. Specifically, if an error occurs during a `create`, +`prune`, or `check` action, borgmatic sends an event to PagerDuty after the +`on_error` hooks run. Note that borgmatic does not contact PagerDuty when a +backup starts or ends without error. + +You can configure PagerDuty to notify you by a [variety of +mechanisms](https://support.pagerduty.com/docs/notifications) when backups +fail. + + ## Scripting borgmatic To consume the output of borgmatic in other software, you can include an diff --git a/docs/static/pagerduty.png b/docs/static/pagerduty.png new file mode 100644 index 0000000000000000000000000000000000000000..c60c63ece018c1c7164c5311be42dd3dd60f4b67 GIT binary patch literal 20107 zcmd>mRa8|`_b-AX5{gK7mlDzq(%mH~T_Q+#r${#vqI83FDxipfbR$T2cX}t^H~!;3 z-1~MP?;b;jLynw%)?RDQUroZ56{Rpx2~iOc5HMt<#Z?gy?##pAFHjKSD}kbA4g5op zm6({ajF=d?y@Q>()oU{Z1o~L-SV8FyW$a)jHHp}Fff5oVZ0dAqpD3erf+^KeYD9}4 z(z09!(qhQGi}?Ik$9~VEa^vqpNpWB?Im(S64=*1>qjdNwO)&0r1Gnxj_fz88rbn38 zG_b}BeZ3t7B&Cqozsj(4XqFtlUJ?kZA+NREYaDBC z;5>PIfskXV8BX-~FD8DU+DT)73%~B_iJg*(k1&;tL#W`3+Cq3N5A3C1IwBz8)4-pO zJ4EBOeejR>on#aw?l0XT!ozt`;y!P9`}mwBw4KE4Y;DcnI3b8Rm>D^lnUcF&Ia!iR z$tWmm`9H)%Kp;nu5r3iXHnpAMs;Ay{@vk3O+oOCa7X>u{HEaUMGeX4#J*_iNJo1OO z!zi_+$+OskZqC!>Vp2_TI8 zpBMb!U-jP?{9j-7fBcyL{i^>z-|YYS^S3{yIG4*--g6TO79Aw4o8NcS9+#HNUv5F- zLpH*NHNQLERl(S55csOXP_>69{w1>E-F zR`#;Kef##syzkxn_vk1ntel*j?Cfe6d@k>jbv}ouv?I)VL}or~Us_rDoxz`Ml%nu~ zE0Rojwy&>mXb5YXawt=u40#s~-KIf^lhe67jt0Ghho7HCr`mGh?2=Wkw5GRq@W)Qf)KoA}Jkhgl*Pmbu%0C_{>OrJ0kT zmJ+o8V`O5wK3k0b{Q0w1nV$RBL}hQUyzxu{;Ude4KnRMcnp(U#65NUDnO7__Yj$44 z#33;S!C32{L z6%_RA-u%gtiT^4bg%_utKRdE3oJXZRC3`4n=j@K;SgdtP<=NgB3z z9DnH-`m9&`=1*fm0lhYto6+`~%cl`{cXzwi^E`D%@9X1n3ChR6I08gicjv>{qdGwq0y$HFVuSW9SSC7zNhq!;s)X!Q+$m$Obq$8d`JV*5R^ZXM# z#$_{Ba=6;xW6HKrDi(v6J-!d$LEwuem6W%#T2Erq%~Q|OLM zAIQYh{%LgXnVQmB=b;R!erlymkNy1l^J1;?F^=u{o@J#hq2N|!0|Nv25jr|L%F6ce zB!-4mGmZ!^0`!>4_?*`E7rR0qkwww{G*l02tZlETs0c>G>k~K>X#?R&Jd1{gC zduRT0O;3L?N{OYCiX!6PnQ3r*`}S?8!5BxW3I0mQBVl4ekLtFzHhJ1WUK04w(z~zl zB^eW~GxommMSi)x9Lo(_@I$)0yWJ1_xYbluVRt*O{wDEgBj>)SlF9RSp-?SfGLrD# z8_a}tyayzD_o)*otA)=3(M%`kJdgGD^o)&-VMidZ8iq@JYz^}g>R@eh-8FYQouT;Y z_~kRfSm9ef0n+IC=MKT?-X7nlr#<0RL#>#Coepatub_}^IB+l7DRl6&6r7xw+nM*5 z%myuMBI6blU%#-(w3V%^tIOhfOL@02n?H{rgxCGR)X2z4PIR9_`tq?8X~*K>4c(-v zV?aZLyNJkFoAQ>V%*>nbc!n{42|A@#Ax2zpW>Qj9AF%zA_5T|il{3gx$0JfN6-E4j zi>>9s%JQ;)gF~=Y&9l2GLqCeHd)MG}QpW!5=rDmoUl)Sv_UfL# z3X;mJ1Z&@$VAE5y5F>F)l{TMiHa0d|rOXoNQ$}7+pPd=rs&7d59&%10{9JaRqs#Y! z!?iIIK6GQs_EJ-G4EC~Sl=LK}a6d0B4Q=5ukht|z|;JLgH4($eZG z5o?2%)RKh9UabJyZ_;hg(;X6ATupU#@rKrtl4mR|{oUQ+tw`$Y$1(Kl_LLD}{A^bVrr!hBjrg_hJROMlB zz;gPv)3>-kC1h}4pt-Kjb@g|~x??r;D)_9Sp`l-Jk`Auy{k5N!m6w+n7e8fSU|?cm zdiLxY6O#r#c4A`Uj~_o)R#rr_2mbu|^ZomG=s$2a+S}WqSV1dfK*=#`Dpvg}T&b<6 ztE&s$BO~K_Z@wcTA)&gu`u+R&2L}!ZiKK!aXBQVCAt5fCV;G32y4u>>FJD$pmkm*i zZ89jgkjbt`LCdMBsWGmsht^(E!AeC%1$E%*)8dj6^f%P;-NeMi&CShMS67HgNPhql z=<8=+Vdl#3B7~ZL%-yvh{}4ODIX5t%tfq!!=5w^BVr6CJ?JbyKF;QVeCHda6!$hqU z6FGHfD7WVw&eM1A-ZAUdf2;{56ZUD=DmQrZ=8g4cJjt2Xa8K}y>R{VUPc?Pf#C$%a zS6!PQ$uUvBPN?#S4ivhwmex+Dn?%@35%vl2b1bi~TUlFAO-vNj?zjoe4(BTs7Zr_+ zWo!^7cAMV2cP}?L7y8-7_32zR3IC5uR=5v*J||04(;NUqO-)U)v9Wg&bw;^v3m!AG zqO7c}v^0;y63TLu@6GvYmhX-G+5p8_|7?q=QaaCDfE6EIUv*UJ7efyKSoG@cQ;PU* zI6Ib>1(A`FiflO?cMsf}4-O7Kefk9Jj)up41x5Dw_;|*~WUS=nXQ}9iC@A(&44s_R zu^vWgU@Z3okofg$h~ssSX%#dfZf@?X52dZGtsg)BovN`3eMrLRypga`$HvYs^&Wq) zR?XbF3ifKM{8*`O-Dt6vVZHr}O+44_DNd6g=vrD@xoOlGXK=Uhxn<*|42s1m_fJnx zkB^tD&ZB2_CKOaOPQ!hlQB!x0j^Y@YiHUVY5^+;wJzyYhejz8<11O}i7P=QLEiJUa z(e37novAMtN;Wn&q|sV>dS1J;g0evZ^zs1SGBPqS3dVC34;&8JNX-h6*(jChVx^%^ z!1lN~B`H)dQl ztoEil9<2>lRk77(Wy2*EO`4GJ=;G4q`>)m7YzVEtwY9qX=xBZTsa9Fe1i9X?e%4;P zEH8)s1&IR`SsaP`zW_9mUQEA_j=nD(Cz}1UzyF`_zukY|-ZCn1{h79FQ_!KEp`@Z( z9Mv++)eSgY>030&ZR8@Q@2$0+Uc;qPKN!lDgRPUR%Fx7`BLrAgw!F;u-{sui8PocT z@(REJf28|EcCPS-s?#(yG{j7AG^lin#m!AkMZP&WIMhpToNP~jH-3)Y>*p{&GjsLr zKAC09WrO2tj;dADfTg7*6p0`Tx#6bgTwEb}pQpCMM3NH|udfdKpU}z=bY$dcN&Y=4B7tqkuSEd}v^R_4NenFBWkL3D_WS zy`oKNFn_V&MnsbeHd+o+wycI&mtU=Yj=lkGy0T)PAt_#eur~PF$HymupOn|(E^{k% z;r9gW^rN$HI0^Nx0F*%2)6 z-;Ft|-xm&Ekw&qxiHU|FksfWS?2CulMCn4_7tjE2mg1G5wB5~o(PANJAJbA<`J9<~ zC-9NbM734Cq7eV+rz#qA06u8A&n95ImH4uui!HpdaR^;k=|=Pio^T&<<;!~edEtNO zN}~skUUzG>e2z!8cH92lj2rsu>*}IK)X5fR(??2CW2L5~SnA6tDk^4j+Kj<5tEr7V zscpc=t;RtOPD~u-p7ZKhehRgL_3!!!a@gXmU-11^$rDrt0bdb)tm)!FuR z9g)}G-+i|^cG9uEIj7iu>J)*=! zDJm|8w(EVm^C!x!KCtCpF4Sm~uBhAsWG_-OGL}Ty(N?Nb&Vdk(k)k?tvX7fH4ea8w z*>n57sq9lai7sC@5aJR8#RcLSHE?lu>KY(4~G}b>?-G!S6~* zNlDINVS}wij+xHmL-|Hc&%=Z7fl1#WSLi+Y3K2SmG;A_5rH7qFwK2*R@EkvI+htZ( zRzjb8;qxTT+|lvy_PpKVv^Jj>jXJ(Pw`PE-O$`l#US~1HHgDY1aryb1%zBcZUu61Q zKXO60nuhYms+2GMGg~b5mE@2Lm#lE;=Lbu;Y^l&r(GSXP8mUsVxi~praMqfhCv2I) zzrqvXxt76W)eyCz}%ROlUVl>eD%U zzSqZipP9^GHhDCzzKo=drKhA6-7J-pl~q+!OHInaGqJlb@OB|+Y!jZV!oz@SJi}0R zMgaKaQe5i(`0^(wC$fR6&d$!3mh>nb68sZU!5=?g+t)ofGEaNy#kq(?=kU zuCp4>gVx5u&28gtA^!2)%STc1ccb&>SHxw~r1w|#($ohhCoZn8t3%3POW`%3n|iH>gp3{*7PdB02#ha2H(RAkQN=?f408}@7mJb4EvBg zo3P_n$pD}R%&-sowX7^xIrdf#Ua2%ni-Q>D8Ztj1xUwH z;zm$Ndw9lkEzGcUbBAKTJbG$3*Q)R^$BA48HoL8*B``-o%vj-_KYtb!7IuICeorw= z#mB#Y6)+_?7ndwGmSxL8%$b+agof;bjjgTV>xl{w1=0*EoHxfxwJX~J67GH%`}7)^ za4^ryg!|!PVS9&%R4#E9eTEMN>{V5V{xrF+kMr~LR&u8Gug*2QJ1+i+^b*Qq9st%1 zs0e5^DJiLSOEqXn&@5pY?mr}+XmF%UH7w2u!NbGL67rrJ9v+^UNLg?p--wBcsqne> zU`8hNviY6P3)N|Rrs0-dCnn;&-No!QehZWv>d}{~D!@@ko8@h6{SPsSIsDA@tq%ACcyqr5PLb-fv20nMpkxRthMd(ze0e@_qT6A+{>$3 z`&n&O)duFyL;CE<$UbP#fQXE4ItBRSXt7cIGX*avEnu~cz6IQSLZ`^_{JF}sd$4GB zL0WqN)*STV(M%KL(~nR99}=*M+oWrkD!H!o5;Riat2Kzj~)v;b*8Tt9w<1ck)N<_h1OQHeR9gYE#wx?I1h7B({ThmTxZ{ zOw56NrL0tT3u~RRKqc5x0#-xRuCA^%FQ~$k1G^Rlp%(#~wx|w9@pB!ku^9&(8WJ4L z%g0ys0nt5U?>BH~*q6ZMM8(8DfBKXMZ{d4$1uW&4sp$BRyZsv4+R|f_BU}sz$^fs= zm%8I&cW1{<9+x;>P5t+pn-gYav_9LLcUUW2_HPk2MeVoX@ zO~ZeLY^GuGu%QB5cLi=vXG=aIO8{^IhmX^W!+rMMJ5-=959c?@2XDeYXomxFykK#2 zcR$|wqdQtMxOAS-vz)=_O!(Y5h&GZI2MEg`g;+?n_|7X?j7dF)6-9U`iOow_TwY`Y z%bmviAbp+e&MEmyJ32bj^R4$no9DKh&9ST&&4xoh0rk)C4$}Mgk=O_S3PYwio}&i0 zkB&M+*}3J3rmiub8N$Aq#94zmdKefOQ1HtO3$sx;mR=_JEbq3Q&XW0FJ$G7Dwy>}; zPLI12o?QXJ=;%3kykhe}XkvH#X8zQh1;Nef&ty&(B|V z_=16zVK}NIp8*>c4J`t+P@9&jk01Tkl`C^NGqi%vnXj9m+WcMqnBJ;F=m&poa@m%Z z89==~ix&Vo_)6&tPYQ{+-+-uVsfx50EhQoGQb(tX{qezc&WQ{|uIf6xN<*onP&P9( zwCm%Gz=@6?==)IN#Z_puw8r~|{}CIr?&FhosQJbzujf>Shj+u-Y`wX#QE0zI_GYq5 zf-x};fmya6rF;o!4*(#hmg0_}SL;K$b6$s$85tz}gvcoALnoebME)(qUqCX3dZb%r zD$|o+opYYa@45s1^hlBbAO8#>eapeGr{VC~yQ$x;kC+Ia81Z}&IH%kSIh)W?4;qk$0bf=ULm&#C2`1v95pvt%10M(`K9@6>5W9WPB3=-qF?y-Zt54LMN z2wkz7`hK5BNdz;i%EV~9CLVok0IF`E5{;vob z8CmY!>gJR`gBlj|bF#2rVWZrYB3C2ev9A4`f~*P zk;r%N4v4XRV?#VVeX_JL0aS4I!^zC3{% zSHp@mO=Q>n+->ibi$DKbs@OtfZYYHbKC-9;<&g^%Q^^NccR!`Z#l?YbV)KDQa1BVp zt8g`D&&5b?bz72FQl;Y2i|f;m$3YK(HQoVT87?cMn>Oe z_e%}=1HDP0d+atJM$Ohz(7yt%4yOj79B1&DDm^w>E*HSm{mC%y6Q5#p4;${g#FlNR`o4u$Tc2~tz%*o>E< zAE!)%G7;7!mj5eA=oog#!h*5v`vic;0VbM-a}S(t37{c^gL9`NauF|A)e6LH;DkCl zQ@|vA|6RMDpa(Dyu%(tjtp^d6lKI|QT~$?F(0gV6wd376vJ>6{jE}u$Nh3gOL0wgC zKW;a|1qb6~Oh2X+w5ie$%mg8m6-I%ee#KqN?7ASEpMeapu($}KO`c61b>J}E->b9i z-gO~RvEhI2{jBr)4YomH;d;n15fESrhC|>{bMC7oa&mHlwIU4I?_j~DBqbfPv}$~z zCsFKh<$4qs7Zdaf%h?7wG|Y5%QZq7c00+gv!AaA3_2R_~fM$xenQl~2-$9Zrvob~{ zv7K#pui(o)x)jyF+=He_D&$3v6+WR^N?W>gc#ai*H%h0#`rVl0G%Nb+KaHbPQ)xNI z0S12npMczP&i5>T?GMMYLj$y^=;%kejK6w%4i68Z-%X-r?~TNeCf~dVE-4oJu%@;) zgnPuHi=KEK8#Ne2`Gw43k>i%7r6s@5Vrq3+Y-X}fP8>WJ7>Pw<O4cQdc2MD5Go;hz%j@E5EWI{3S{* z>qEimm%c`Ki)ktiF+5ez095q^gZPvbfCe33zlvmpoEH@_mfk;VJzGG7tHxWl zpZ%G~U5_zqts;XqIPE|;7-jIU<`)+~r)k79js0C1kP%nUXy@YMs;#ZH<|HihI#_zp z{AxjEki%Iy7vv+I!_|%K!I=9tK+-`&ALsOA>=fqpzTog)qm1$q!-;VzN&TNJ0IAQf znO|~M zC6U%ccb@Fo+6M((5Q_V|cb}rKK?3Oe{d;At)C^yeHhy_&32;g1fkB>p>caedc6Z=- zYgA)9I;gR3V5vdnde~j^t&H=8jF`LlOL1W7Z|Y03XVUFE+~3&oL*6<&2TOaVG2%JQ z(nmIeR137`hz{ey1Alh}lQpj=%J>e5kxjV|o=ex5Gon6v#LdPQFdt*5>?V-e(jpk7 zo*R>uRjy`9R1XR`BJy{VT`Hrlvk36_AEPsi2U{;ad6` z&GSSb-Xm6;qs8QG2F!(_p{iqZTLDr~gf>p+d{=q8gYU;mQ?IYDC-ek~&L=RaRsY7) zxj{)V9+oRu9|vOkvE7imFu*PH=zgfQUBRG^F3Q#B-@g@3>qAVswaR6oXZMdR1E^0L zY5OUR0Jl}>{It{-ep0ku+#mn->(@I-$U(uukp%3>qS0~QOpRR6M*Uo%<$!PwvO4%5 z;3k8*q2=Lm0rY9nzuI%AM0VcnHx|yJ>p2iLXE2CIC8e%+9FRVZ#kI)r;uL75#T;}A}AlK zIIOU}W@l!a!S`&O9Bgv68?aQ^l`i08WACTI`3|KA&@*7kwe4*ZOiToJy?T~O#eB6L zYZrYj5Jc706A}~6{-dF1@;NiIutc-9nU8D>%p3aygC50G*4J0y4ZnMT_9uEZ=ghqd z=Re=Ry?7KEQZ1sR>qumM1mYC|$zfVY0+~x-GdWuyy&JlcnvT&)VEncceJ61uA3eQv zan>H_&Y*YW;^H1z3IJ6;>eU9=>+I^9-;<$iWeS$-A>1e^7c#Fgsuq-t<*J0KcU*EG z$z+8FlSA16;q|t7*w}z)0Dpm|0A#?^)6Fnfb(^4{Ru6>fK{wX!sADu5r8RWcu%n)shs})@qK&*JDoOGA`Xye(LbP zcuOXAETW_Gq{7R5xAKtpIl3e0@Bnz}XlaYyNKhL@Xb$0x?&Vxe zhWB?7=ISYRpHKVz{PhbsEeAXMFem0FTJ3TfIHBAuS~ef3r$aHBYfyV$wRoQ98NP(4 zm72P~v?Tu4Nt!xtxAijWfC_~5z0=ci>qOLUqTmS7nlPHKN~^z!I!h1Q^&`JsLA@IQ zQHV|8nRqM)K~MHQTorQ#sXc7?mIj7QGdp-6?=jT;$TnRtFDxi<^S>ODYn{hyKXiC0 zlzaDw=bP$Pj#k+kREe4|UvwKBXgYDn#6?9#O--rADYV#fzuCjKW0s*W)5HPQffy6> zqsE;x*~rdE&cc8Xt`oLALt?Y9Vm;5DcO+fGWe~m5a+?p89(iiMC1AR#s-; zt@0Z+a6y(2tYa@A6Ra&=gHr{v%11*1u#5m8aw(Zl_-op{5G*Yz zf$)e7_X?0jLPA3I_>I^$3XxD+C26=o7^!!A79{)QHNV}3_{V#QFH}?p<7ni3&X@5* z60dY^bdB<}5&e<1(6{=ks;iwq%>f+s_U-=33FsadAZjsW?~X)Cf{mHLQLpR=B7IP^V}U_B)G7rAifDwt4F*qXUo0POO~$*|suYzra^cgE6nMPQ z00*?|Jwbd-jL15z>&^^;fEz16D%FHLqW{O%Rv(bR5;UiP((UcpGb4vqAk`=?EMTJC zeMBat*XU#=DvG$}`B^@1LU-*i8$fh`F?4ivu=QgW6+sJtuv1SDgfXUid*7%332>sG znPa}1-`YZ-Tw%z_2k#R=1LsQik~_o`eyw5g;0Ndol)rU#-JPyu(L)Y2)7Q^XotV({ zz623%WCUB2+(1+<4gD)<)fJ=?`5=p31ZzCsD$!9ZI6ZmB2EeVn`QTmN&Z9pY;6r>I z?2MIboX(X?CU>aQkqV8BRM6H=iHyWZW>AzGvRnxK-P02h5g{in?T0XYWQ8^oTT|l< z+UfQ6wYQfSF$oELZEk6?g>9dUiEJoqWJDppG$2C;${~luIr(l0NYh0|0;uMViqb|bOcIlx#~U?3TR`eN;M>^T%vwE13B$itdeU4O(-CO z0##7gCgR@6g@bL>yB9`Jo_QfZe?m&)C?u$bH%p)WO84YOhOP#(w!5VaSMxIpE)TWRJNcgt(GnNXFNaycB;$LuHnIY<6oBMIf z`{F44-MfXIKrF_V8`!ZREa5}DvJ$8?-QL~?5ByGdai*>o#Sc3d7p~V!10zmBqDEi> z0fTO}pBIH*Ry$gb<6G`RxcL2Y-Mf3Uc^zU=qnXucy9TFr1#rlHR{O**b7zd}wuo3wae z5{IkFY7VPWAR64L)<5RoJg%l2I7yzwY29B1O_ zP3PVlrGOLtOt-cdq$I0vk7bp$w78+{?CtGw4O)haJDeYwdGyKV4jFZXFlkpH1f)OK z6>IyXsin19G^o7$`0-;|oZ_U8&_|&mA)>D$ld>%Qh4j?bkIH?o<+RXHcAjE|lb7H~ zbUh;B;{YVclOM=-JwuIDq{Em93BdH(vcHFr;z|ARq4*7v-yDSyrh+i+^d~>_OR`85 z1iqkm=*czzaO&+`!t4=ZWc&poF18^(@J2ISXCI>mD^K6zMn%Ojb-K>MZ+}deo*t{F z5Ql#U*F>)$QOF4Y$IqWA#tH)czaKnAZP0tzy+hw6h95EsMr=(D=Ok)r*GLLS-q6rc zdOC5&uLt}?5GcWYrv35w;?}~0L<@Dkj9f2n)t$Gn>ag2AsPC=b=9xaKUQMTPcVTVfdOcykHSxUE$`V=^lxcNCOe^Y&F7;tv^AID* zbfBW59-TN}v7mj?d;9bpclgfuqp&weo8ylZk*te~As?#!DKt<50tncrzYmssV55Pd zf7;ri<*6n6>#2*dd!`1mJ0in+S+*z@B_-ru!bR$qZUHKsWLCEI9H+f5C?M-KceSEO z1=F6ftkDT}V$fitf@I=;b@o;;GozEIuCg+f(+2l?0m4J!31by<8R*CIQ*v&sxt zl_6xebG$jO){c$LfjzDQ@>+Gxq zFK#NTK&;C|Rh7R<&)L{?YF`sv%d+~|&D8Hh!~z=^x9ZWZqwiz#9oTl@L-3vj=nPRp z*MV-zc~6ci@eu47PHt`@>W4_ryT2RnN0IPDEEEY55x(AD;JCw;^%yke9+o!1j!EKb z>+~D)20P%SLmuZClF{R%qgK~DF%)8w(WDA&zY9(f&&&X0pU}E$fS&-87HGaak}s9- zh1hG)?|zIYty*q?hVWT3QuUI&^G|GMrh%bhIl~oS_c*$@9-aS&bSlF{=mJC0IF=TA zs*{$cCM^w(3NHF&IFY*W{W>2vx3f?(AL*<%xm;B$s?SLGA3_jt5p*!9=Ud{2vVQ`% zGpL_80fSV`5^AxZZ-ngXbK&ONkCL99)+!uIjwSfA@Keq&2j?;Uq|!~bxqA8=ZB(3^78VtK9{yD zFOBai2;b{|#G>Cg0x?bq#?Q3*l5sKJDat1=TJWpv|xzW$)~t{ZBj#=0v!6qh@s>&TJ_XhyjhmJ z=f?Oe5VZh!k;dytyEOK>&cY4t*7^;4@C1Ks%vphHczRk_t=+TzHbZv2G1}P3heZe&N)W zrt$`E4~q*C)h;OEO}8nmBRLJDxNcLRTeqnqsF^h(>;Cd0fRIev^TQ!!lbLR{GL$_L zo(T!NJM2t>%?wd2EoEg7jZ+CK+(r7U>go(Djqfd2K}G4v)KY+t(goPMu|n~W_=L(` z!n_Ii#y`mFt>f*UTlH`BYd%pE-Mq0p;LCo?4Zb!!av+;}v(sKQ9rNO{%0uuthYZiP0k|E>)3|c%mwF!zgK92cHp`!1cm6Vj^$|aHV`^c*y z;MU*#ah=0JMWsv|n$=6+suPiXrf|bI`cK2a00Ide0Ce*J^?=DUi%`3yqC>7e1->JU zN&FtErv@)oXWR6BfJjH&!@p(3b&tstQY zp$_C3C-GVV&{(gbP>rZ#DNOVF!5jfEKflephn+GvAULZtLd;aR?#<}ZR^S{IL8uqG zyC$4G4h?#gu^gP7{_e$zuq_y|-@*FSNTzNi=|X#dx}sWm1Q~`E15c15Kwh%8K@##u zG_|m3J74)Q(^9adC2gz=nhK!uO7?du>aPH6u=ZL~!dwXxlS0kXtGVvwgC9R6Cxi-W zdLiNpA09YSNI8_RLUma#@2A&t%zo1lh0cX+kKT^MeEU7stSz<9>aSA5Fg=bh?I^ z!4oX3R}4v=OrKJ4{*yGu#lvIKuJ{X&ob|Os>R4<8>=y+AIxr?6jQ;%{4lF0Qk}Eso zROem*YA4&2_OIHHVOHRBqqrQL5#EPGvbvT=4s=-ILU6EPU|>jT|Lf3Lm06DqbRRem z(FE+0)L60W4MnNQq+)=FwYAS6cn1--^Q0GeD2LK&?iWW0M`n|Ag^{PxC?9W$~ z$_Zr|YH8hnZZ$kKG!lYG7s4p@MTI%p)7@Ra)hl(PL9ds?GXCb~2Ds}lAdu@AasLTA z(Szm*hnkWSQ4=j8kH2#WJL3C;2M++cnW1}E#5{Ah-o61R{1B2sPEHB3xm*5*h4`Bb^ z2xOEC3c^f7O@*7rP>+v}AU{K~SWC-Kf(lVVqzZZ62UV%m5TamA1V>xz5x-8igMW+K z3~hEdNS;Diuk%ytDK0Bm>mrEdn4N06#7IGp}JsOwb~GmV|s6Ohfz& z-bVYZ^RKrE`2J>WFCjUmr>6&b5XkKI2t?2r7vSK;#6e9d(`#U(r2GUSHj`d<1(PNb zatMfk8mi;&ehyZq61PFvrAD?griBU^`}Xz~AYyPn%4^Vn2j`y8tHkiO3 z5SSp(`<%xM388)Pec>>+K{NaL^Cze{{?msLH~{MuN`waoMU|q7mD%}vepZ=oU6g0t z@#!fQ}{Z!)F7rW0pBTZmwReC@JyU25tiIAYb-)k9d6TX|sK1o^1pUq}D=967yzONY3Z!ZNZqixX)<2_4?hzCegbeBMZALdk zi2hzQtIc@+xwaD(^H2|A zOJbI4w|!kJ>X?VQpA*PxiY9sN48L(1KpY*HjZ%YtM#IQhR9icytVj)kMcBSJ+iztB zr<7fxyTGO;4z&&zeN0FwTg$tqAoXtiTU5kt&0DKN4aaKc$$t!3;fRQcu>WFHZm)Wy4&;g8A16gmQ)9Lg7hp1v z7N!~jDw_Ow4@R6>=yMo2*7@Sq*w`rQK^TPtFbOg!iGYM0?{Uz;rsajJ63w>@87@k^YfdV(}ikuAb0wUh)7}|uyhkC;PNuK za3s(EuFZ27FfcW}O{}|uuWS4rRaUx~ zR5L+NtG>P-vLeoqi^Xh_*rtx16_Xym9Z6#~YKPexav$;z)Nz?q6^K8 zpxKOGY=0;a27M<#A7(DVWlVoSgu|2hczNhSq8`mQ2oVeqnmwR{+Sq{a_xCnO1*zeU z(Vkzw-s$JQPUR~3{#lv+RzZLBXJd8s6h>pe)Yqq62k=~&hP62Zu7Sydn!37B_R#mg z-r!!3#M@@^^75AIH4M$p`t&3*Rp`G8LSKF{l@lMY1}i=G>2{LI{0Gwv9=T}~ zB7Vs2hy-=1wWApX=N+(%FsAu*VKxus#M?OmW(dbFbcSD^?gDfMU~k2_BmEC1W8lJg z_|Rv&W?Y20yHcwjxY6*pZ+EUpi-c7n=c1~LSm`t2@Ui&nu^s>6Y09Z}aESL%#-BT3 zJO6NoOTHi=pzGiMUVxKny98sW?I|4Nuki+*g9P`NgS><0ARGfm-Qwb6d|VtU$RRND z$I!t`nGEUQJ_;-=$R5MPr3kot*;_e=EV-dakIl!={~;xXzDE8`mogUSPwl<56%>LY zgR7$rG0U0(3K+hHk?HX8aB1o)=*e(F4?H?Tha5Oa$c~O`gyG?M1)LtlR*C>7dwaJa z9!LmkB(PPu2@v1vsow<#z?e6rNF*gBVCk5v~WN$RNh^?^a@UTvY^h$I7a3jCTCM2uCNjt?gfd%I7trUf5Tt=jFN+uiReG z2}gJJK%cVg*kFz&Z?=ZsCP| zrw3E=zwRVBGZ1A1{C2uAigtngz`mol0Z_oLMtVEP47or`6)+NDQf^X4+lljff{W^O zS*oTqrhgTTSeRc|^|6KcCCK3z=;$NPxIfAqCrF04^T!RK9p*4Y46OLl`Rc73Is8&=%}f4e zTp$~Tg@u^QMKozB8|ZYG+gn=+K9M=!hX~I*{jS3Vvx5$1fd4{z1xOiSHn>3D1a2>D z=;Yvlih{BSAt+bZ)5F7vsHh#VQl?I|4GsUn^a%sWm5H1eBx8Ve^12Xfg!V&%8u9=T z`F|G?0pt%B3)m>kYRtPp=Kwnxz7}hiZa{BG6MVFHJTfu@m~#nsS;A8={4>(ik0Lnp z3_kNyZmryNmfH9Pn+MPmWG`SNfsRWV`*jaQWN`FF3PI8Pc0fs$6L&Aai%0xjLUQu> z#Kg~s)8$U^F;e5>pFGPWGW{C3!-S0r14Mjp7sO?07vbGZVVnf+#G~^(XT%*!0<0Ir z8q&4>gx5U3lQ0o|jO%DlJ$$Us;c~#}6^~MKZ$m^y&N&)q@CYE2{^kzrjlHKK)ep zmzc-?)l{_=_7iImB!E_G^3QDSMz7!_*6ZiSnVkC83S$#up1JTUCQW|eOGt0lIaH=+ z;IIC0t6jh(4lHu$-awJb7N|L(g|I{Fqx2hf15%<$oS`LSZCxGr^xk;WZ<#$ytfl_p z;rO_hdfPMco;mIZS^EV`*$Zh7QXsb(gBp5Stgf?SOQS)7 z7mtfV1FJAjh}RTs7t^(fgNG-EDE4btY~+4mfmv7W=Z0z8G=cBPWDY+kZXmhddNO5m z6ajfSE{`Kiwy)JW)TZ{0-j`h7FQ_1-0n~Q`KyZ(RQQAhj9QA|?5y=CiHrfj`4V&TW zP(&h^7|ji1ctNOOa2h^vVL((s^9jVL(Xm=%nB;5z*<3VnzQJfyVyi1E+j{(D5p=|` z_wS)6Q(X4y#LdMU;it-<0N1x>`!wGH1s*gG0xz$Hf(H6uBn9`Mr*}jBlw{c z>3j)kz1(Sv85$>-F*=(d)?IY|Wy%H|_UG?>C;JdVGoiN&ozOzQ4v%B`hbR(F!@Xu!decT6KF>HhQv5H-|=^1@T{F|CI?|`E0TM5z~dS^ z5$vx-2k~`|b4rFgx2)^m)#~kdVS?#rnA6>|S3GOQ43fh`x^He~mOD1_)6njkP;rlt zF@Ve3srV-c4>ORffyU}Ko= zhv5WEaVnH9<{tt*sL&h)!L3S2z^c(_Wb!bKv z&;()(gH%ZC)Lq*2M&T2@_!ORBA+LD4J)PhE@1E+;>MA$nyZ?FadwCtBxM(aR2)&-5 zBZ_$U4jCQa9`c;o%F@r@3SI4q(OJ-Sl>IP~vc=&~`&nIH&hmin(}{7_=T^{7+pZ3J zK&6>r(Y7t&Ur>R-4g^D>EN_gJPDAx)D82ut(q&r*m@dynaQTB5>PSY6_r90F4|(?N z3N#XM)}geJ?+4R1+?o{-ULhu9H>EKA+2Xow5eC0S0oD}^J&0$Z(5PeV+d(kh90v1Y zoudHOadB06ORZacza6@`y}&wzVNl396NF@Ua$%u@KYBz4N-}(G82_0Xedv1E>X59f`si@L@@x@o85lWg_WdU$W$37_ zJpsrajBS9Oy?uSbq7+&EcF_5+Kg?r)}nofm<~+kcwIq3bGN z`eQELLolxd#-jVh9D}&&o-J7B5bYW#L8c&7tX+-Q40+~caJuPxYH*+vH1s?JV&{9st zuW0*%HKbdM6+i|d0XCl2tYIse(%dXCIyd6s8pij7wHMMgVL*|c8F&#KO&2vkgZnxFLbJ!@e5c~r$Z$x#! zE1?`Ez<%A#o_QVwDHw?j5iZu$9U@YK0Cmq(%1kZ zZ@{~6;HjZuV8FO_+QD;5luv5oucZE@a##()WHChI=@VsjYa+6RFL)yx|7raZ&6cO# zgaL3cMO`;W3gNA8_BtNP5*^gLrE8S?x507~L5j_|6XmC5Nm>cY3ru~_GF9akcz_D& zJOJ=;5B%TCQw9?Lc^+bea6GOEDO97v--gTGsR7L+d4UiN3mtgn>Il;lP zOYcJZCf|{>G=vM9{)_?KTmyO{{Bjuh#UjKaHlsnv^`~V_b9~L?&{qFzAOdUDYxBv- zq>fa8HU*0WwE^N4prpWb&?{-~vxf?I-TnC&;aJUiY%={a1Gxdn^X_Ou+y0@~2% z0R6-{$*yxhzMG8@vIp}uzq5oNz1;kH^1o)?>hus;_5w$PfxF%zAUXMP-?A@vc1dKF z?B2Lia$eKV_*D)!ji>Ab?%-r$V$2a)ctpV!SOT)Q>@d&rZ~yqS26)o4E?S!Ys0QdlzI z_8l{ub9>wFg`m@@w#0wn;YK}#3UPW8(n(aKXH1cQRM(z}KlOhnHOeoZHt!eklrjcS LS3j3^P6*v literal 0 HcmV?d00001 diff --git a/tests/unit/hooks/test_pagerduty.py b/tests/unit/hooks/test_pagerduty.py new file mode 100644 index 000000000..76c5451a1 --- /dev/null +++ b/tests/unit/hooks/test_pagerduty.py @@ -0,0 +1,35 @@ +from flexmock import flexmock + +from borgmatic.hooks import pagerduty as module + + +def test_ping_monitor_ignores_start_state(): + flexmock(module.requests).should_receive('post').never() + + module.ping_monitor( + 'abc123', 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False + ) + + +def test_ping_monitor_ignores_finish_state(): + flexmock(module.requests).should_receive('post').never() + + module.ping_monitor( + 'abc123', 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False + ) + + +def test_ping_monitor_calls_api_for_fail_state(): + flexmock(module.requests).should_receive('post') + + module.ping_monitor( + 'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False + ) + + +def test_ping_monitor_dry_run_does_not_call_api(): + flexmock(module.requests).should_receive('post').never() + + module.ping_monitor( + 'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True + )