diff --git a/dev/CODE_OF_CONDUCT.html b/dev/CODE_OF_CONDUCT.html index 728738f2..f9f9789a 100644 --- a/dev/CODE_OF_CONDUCT.html +++ b/dev/CODE_OF_CONDUCT.html @@ -1,5 +1,5 @@ -Contributor Covenant Code of Conduct • yardstickContributor Covenant Code of Conduct • yardstick diff --git a/dev/CONTRIBUTING.html b/dev/CONTRIBUTING.html index 30c5c574..3f10db2d 100644 --- a/dev/CONTRIBUTING.html +++ b/dev/CONTRIBUTING.html @@ -1,5 +1,5 @@ -Contributing to tidymodels • yardstickContributing to tidymodels • yardstick diff --git a/dev/LICENSE-text.html b/dev/LICENSE-text.html index 6ce9f14e..166b0b06 100644 --- a/dev/LICENSE-text.html +++ b/dev/LICENSE-text.html @@ -1,5 +1,5 @@ -License • yardstickLicense • yardstick diff --git a/dev/LICENSE.html b/dev/LICENSE.html index dc9a9b8d..252d8bae 100644 --- a/dev/LICENSE.html +++ b/dev/LICENSE.html @@ -1,5 +1,5 @@ -MIT License • yardstickMIT License • yardstick diff --git a/dev/MAINTENANCE.html b/dev/MAINTENANCE.html index 7ca0955d..bccf72ee 100644 --- a/dev/MAINTENANCE.html +++ b/dev/MAINTENANCE.html @@ -1,5 +1,5 @@ -NA • yardstickNA • yardstick diff --git a/dev/articles/grouping.html b/dev/articles/grouping.html index 590d7472..e517a6df 100644 --- a/dev/articles/grouping.html +++ b/dev/articles/grouping.html @@ -14,8 +14,8 @@ - - + + @@ -93,7 +93,7 @@

Simon Couch

-

2023-11-19

+

2023-12-01

Source: vignettes/grouping.Rmd
grouping.Rmd
diff --git a/dev/articles/index.html b/dev/articles/index.html index 3f6779b3..fd7e29ce 100644 --- a/dev/articles/index.html +++ b/dev/articles/index.html @@ -1,5 +1,5 @@ -Articles • yardstickArticles • yardstick diff --git a/dev/articles/metric-types.html b/dev/articles/metric-types.html index 78f0db1a..47889a5d 100644 --- a/dev/articles/metric-types.html +++ b/dev/articles/metric-types.html @@ -14,8 +14,8 @@ - - + + @@ -93,7 +93,7 @@

Davis Vaughan

-

2023-11-19

+

2023-12-01

Source: vignettes/metric-types.Rmd
metric-types.Rmd
diff --git a/dev/articles/multiclass.html b/dev/articles/multiclass.html index 877f4fc5..eadd5f8c 100644 --- a/dev/articles/multiclass.html +++ b/dev/articles/multiclass.html @@ -14,8 +14,8 @@ - - + + @@ -93,7 +93,7 @@

Davis Vaughan

-

2023-11-19

+

2023-12-01

Source: vignettes/multiclass.Rmd
multiclass.Rmd
diff --git a/dev/authors.html b/dev/authors.html index 0d37a0bf..36edfbc1 100644 --- a/dev/authors.html +++ b/dev/authors.html @@ -1,5 +1,5 @@ -Authors and Citation • yardstickAuthors and Citation • yardstick diff --git a/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js b/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js new file mode 100644 index 00000000..e8f21f70 --- /dev/null +++ b/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v5.3.1 (https://getbootstrap.com/) + * Copyright 2011-2023 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):(t="undefined"!=typeof globalThis?globalThis:t||self).bootstrap=e()}(this,(function(){"use strict";const t=new Map,e={set(e,i,n){t.has(e)||t.set(e,new Map);const s=t.get(e);s.has(i)||0===s.size?s.set(i,n):console.error(`Bootstrap doesn't allow more than one instance per element. Bound instance: ${Array.from(s.keys())[0]}.`)},get:(e,i)=>t.has(e)&&t.get(e).get(i)||null,remove(e,i){if(!t.has(e))return;const n=t.get(e);n.delete(i),0===n.size&&t.delete(e)}},i="transitionend",n=t=>(t&&window.CSS&&window.CSS.escape&&(t=t.replace(/#([^\s"#']+)/g,((t,e)=>`#${CSS.escape(e)}`))),t),s=t=>{t.dispatchEvent(new Event(i))},o=t=>!(!t||"object"!=typeof t)&&(void 0!==t.jquery&&(t=t[0]),void 0!==t.nodeType),r=t=>o(t)?t.jquery?t[0]:t:"string"==typeof t&&t.length>0?document.querySelector(n(t)):null,a=t=>{if(!o(t)||0===t.getClientRects().length)return!1;const e="visible"===getComputedStyle(t).getPropertyValue("visibility"),i=t.closest("details:not([open])");if(!i)return e;if(i!==t){const e=t.closest("summary");if(e&&e.parentNode!==i)return!1;if(null===e)return!1}return e},l=t=>!t||t.nodeType!==Node.ELEMENT_NODE||!!t.classList.contains("disabled")||(void 0!==t.disabled?t.disabled:t.hasAttribute("disabled")&&"false"!==t.getAttribute("disabled")),c=t=>{if(!document.documentElement.attachShadow)return null;if("function"==typeof t.getRootNode){const e=t.getRootNode();return e instanceof ShadowRoot?e:null}return t instanceof ShadowRoot?t:t.parentNode?c(t.parentNode):null},h=()=>{},d=t=>{t.offsetHeight},u=()=>window.jQuery&&!document.body.hasAttribute("data-bs-no-jquery")?window.jQuery:null,f=[],p=()=>"rtl"===document.documentElement.dir,m=t=>{var e;e=()=>{const e=u();if(e){const i=t.NAME,n=e.fn[i];e.fn[i]=t.jQueryInterface,e.fn[i].Constructor=t,e.fn[i].noConflict=()=>(e.fn[i]=n,t.jQueryInterface)}},"loading"===document.readyState?(f.length||document.addEventListener("DOMContentLoaded",(()=>{for(const t of f)t()})),f.push(e)):e()},g=(t,e=[],i=t)=>"function"==typeof t?t(...e):i,_=(t,e,n=!0)=>{if(!n)return void g(t);const o=(t=>{if(!t)return 0;let{transitionDuration:e,transitionDelay:i}=window.getComputedStyle(t);const n=Number.parseFloat(e),s=Number.parseFloat(i);return n||s?(e=e.split(",")[0],i=i.split(",")[0],1e3*(Number.parseFloat(e)+Number.parseFloat(i))):0})(e)+5;let r=!1;const a=({target:n})=>{n===e&&(r=!0,e.removeEventListener(i,a),g(t))};e.addEventListener(i,a),setTimeout((()=>{r||s(e)}),o)},b=(t,e,i,n)=>{const s=t.length;let o=t.indexOf(e);return-1===o?!i&&n?t[s-1]:t[0]:(o+=i?1:-1,n&&(o=(o+s)%s),t[Math.max(0,Math.min(o,s-1))])},v=/[^.]*(?=\..*)\.|.*/,y=/\..*/,w=/::\d+$/,A={};let E=1;const T={mouseenter:"mouseover",mouseleave:"mouseout"},C=new Set(["click","dblclick","mouseup","mousedown","contextmenu","mousewheel","DOMMouseScroll","mouseover","mouseout","mousemove","selectstart","selectend","keydown","keypress","keyup","orientationchange","touchstart","touchmove","touchend","touchcancel","pointerdown","pointermove","pointerup","pointerleave","pointercancel","gesturestart","gesturechange","gestureend","focus","blur","change","reset","select","submit","focusin","focusout","load","unload","beforeunload","resize","move","DOMContentLoaded","readystatechange","error","abort","scroll"]);function O(t,e){return e&&`${e}::${E++}`||t.uidEvent||E++}function x(t){const e=O(t);return t.uidEvent=e,A[e]=A[e]||{},A[e]}function k(t,e,i=null){return Object.values(t).find((t=>t.callable===e&&t.delegationSelector===i))}function L(t,e,i){const n="string"==typeof e,s=n?i:e||i;let o=I(t);return C.has(o)||(o=t),[n,s,o]}function S(t,e,i,n,s){if("string"!=typeof e||!t)return;let[o,r,a]=L(e,i,n);if(e in T){const t=t=>function(e){if(!e.relatedTarget||e.relatedTarget!==e.delegateTarget&&!e.delegateTarget.contains(e.relatedTarget))return t.call(this,e)};r=t(r)}const l=x(t),c=l[a]||(l[a]={}),h=k(c,r,o?i:null);if(h)return void(h.oneOff=h.oneOff&&s);const d=O(r,e.replace(v,"")),u=o?function(t,e,i){return function n(s){const o=t.querySelectorAll(e);for(let{target:r}=s;r&&r!==this;r=r.parentNode)for(const a of o)if(a===r)return P(s,{delegateTarget:r}),n.oneOff&&N.off(t,s.type,e,i),i.apply(r,[s])}}(t,i,r):function(t,e){return function i(n){return P(n,{delegateTarget:t}),i.oneOff&&N.off(t,n.type,e),e.apply(t,[n])}}(t,r);u.delegationSelector=o?i:null,u.callable=r,u.oneOff=s,u.uidEvent=d,c[d]=u,t.addEventListener(a,u,o)}function D(t,e,i,n,s){const o=k(e[i],n,s);o&&(t.removeEventListener(i,o,Boolean(s)),delete e[i][o.uidEvent])}function $(t,e,i,n){const s=e[i]||{};for(const[o,r]of Object.entries(s))o.includes(n)&&D(t,e,i,r.callable,r.delegationSelector)}function I(t){return t=t.replace(y,""),T[t]||t}const N={on(t,e,i,n){S(t,e,i,n,!1)},one(t,e,i,n){S(t,e,i,n,!0)},off(t,e,i,n){if("string"!=typeof e||!t)return;const[s,o,r]=L(e,i,n),a=r!==e,l=x(t),c=l[r]||{},h=e.startsWith(".");if(void 0===o){if(h)for(const i of Object.keys(l))$(t,l,i,e.slice(1));for(const[i,n]of Object.entries(c)){const s=i.replace(w,"");a&&!e.includes(s)||D(t,l,r,n.callable,n.delegationSelector)}}else{if(!Object.keys(c).length)return;D(t,l,r,o,s?i:null)}},trigger(t,e,i){if("string"!=typeof e||!t)return null;const n=u();let s=null,o=!0,r=!0,a=!1;e!==I(e)&&n&&(s=n.Event(e,i),n(t).trigger(s),o=!s.isPropagationStopped(),r=!s.isImmediatePropagationStopped(),a=s.isDefaultPrevented());const l=P(new Event(e,{bubbles:o,cancelable:!0}),i);return a&&l.preventDefault(),r&&t.dispatchEvent(l),l.defaultPrevented&&s&&s.preventDefault(),l}};function P(t,e={}){for(const[i,n]of Object.entries(e))try{t[i]=n}catch(e){Object.defineProperty(t,i,{configurable:!0,get:()=>n})}return t}function M(t){if("true"===t)return!0;if("false"===t)return!1;if(t===Number(t).toString())return Number(t);if(""===t||"null"===t)return null;if("string"!=typeof t)return t;try{return JSON.parse(decodeURIComponent(t))}catch(e){return t}}function j(t){return t.replace(/[A-Z]/g,(t=>`-${t.toLowerCase()}`))}const F={setDataAttribute(t,e,i){t.setAttribute(`data-bs-${j(e)}`,i)},removeDataAttribute(t,e){t.removeAttribute(`data-bs-${j(e)}`)},getDataAttributes(t){if(!t)return{};const e={},i=Object.keys(t.dataset).filter((t=>t.startsWith("bs")&&!t.startsWith("bsConfig")));for(const n of i){let i=n.replace(/^bs/,"");i=i.charAt(0).toLowerCase()+i.slice(1,i.length),e[i]=M(t.dataset[n])}return e},getDataAttribute:(t,e)=>M(t.getAttribute(`data-bs-${j(e)}`))};class H{static get Default(){return{}}static get DefaultType(){return{}}static get NAME(){throw new Error('You have to implement the static method "NAME", for each component!')}_getConfig(t){return t=this._mergeConfigObj(t),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}_configAfterMerge(t){return t}_mergeConfigObj(t,e){const i=o(e)?F.getDataAttribute(e,"config"):{};return{...this.constructor.Default,..."object"==typeof i?i:{},...o(e)?F.getDataAttributes(e):{},..."object"==typeof t?t:{}}}_typeCheckConfig(t,e=this.constructor.DefaultType){for(const[n,s]of Object.entries(e)){const e=t[n],r=o(e)?"element":null==(i=e)?`${i}`:Object.prototype.toString.call(i).match(/\s([a-z]+)/i)[1].toLowerCase();if(!new RegExp(s).test(r))throw new TypeError(`${this.constructor.NAME.toUpperCase()}: Option "${n}" provided type "${r}" but expected type "${s}".`)}var i}}class W extends H{constructor(t,i){super(),(t=r(t))&&(this._element=t,this._config=this._getConfig(i),e.set(this._element,this.constructor.DATA_KEY,this))}dispose(){e.remove(this._element,this.constructor.DATA_KEY),N.off(this._element,this.constructor.EVENT_KEY);for(const t of Object.getOwnPropertyNames(this))this[t]=null}_queueCallback(t,e,i=!0){_(t,e,i)}_getConfig(t){return t=this._mergeConfigObj(t,this._element),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}static getInstance(t){return e.get(r(t),this.DATA_KEY)}static getOrCreateInstance(t,e={}){return this.getInstance(t)||new this(t,"object"==typeof e?e:null)}static get VERSION(){return"5.3.1"}static get DATA_KEY(){return`bs.${this.NAME}`}static get EVENT_KEY(){return`.${this.DATA_KEY}`}static eventName(t){return`${t}${this.EVENT_KEY}`}}const B=t=>{let e=t.getAttribute("data-bs-target");if(!e||"#"===e){let i=t.getAttribute("href");if(!i||!i.includes("#")&&!i.startsWith("."))return null;i.includes("#")&&!i.startsWith("#")&&(i=`#${i.split("#")[1]}`),e=i&&"#"!==i?i.trim():null}return n(e)},z={find:(t,e=document.documentElement)=>[].concat(...Element.prototype.querySelectorAll.call(e,t)),findOne:(t,e=document.documentElement)=>Element.prototype.querySelector.call(e,t),children:(t,e)=>[].concat(...t.children).filter((t=>t.matches(e))),parents(t,e){const i=[];let n=t.parentNode.closest(e);for(;n;)i.push(n),n=n.parentNode.closest(e);return i},prev(t,e){let i=t.previousElementSibling;for(;i;){if(i.matches(e))return[i];i=i.previousElementSibling}return[]},next(t,e){let i=t.nextElementSibling;for(;i;){if(i.matches(e))return[i];i=i.nextElementSibling}return[]},focusableChildren(t){const e=["a","button","input","textarea","select","details","[tabindex]",'[contenteditable="true"]'].map((t=>`${t}:not([tabindex^="-"])`)).join(",");return this.find(e,t).filter((t=>!l(t)&&a(t)))},getSelectorFromElement(t){const e=B(t);return e&&z.findOne(e)?e:null},getElementFromSelector(t){const e=B(t);return e?z.findOne(e):null},getMultipleElementsFromSelector(t){const e=B(t);return e?z.find(e):[]}},R=(t,e="hide")=>{const i=`click.dismiss${t.EVENT_KEY}`,n=t.NAME;N.on(document,i,`[data-bs-dismiss="${n}"]`,(function(i){if(["A","AREA"].includes(this.tagName)&&i.preventDefault(),l(this))return;const s=z.getElementFromSelector(this)||this.closest(`.${n}`);t.getOrCreateInstance(s)[e]()}))},q=".bs.alert",V=`close${q}`,K=`closed${q}`;class Q extends W{static get NAME(){return"alert"}close(){if(N.trigger(this._element,V).defaultPrevented)return;this._element.classList.remove("show");const t=this._element.classList.contains("fade");this._queueCallback((()=>this._destroyElement()),this._element,t)}_destroyElement(){this._element.remove(),N.trigger(this._element,K),this.dispose()}static jQueryInterface(t){return this.each((function(){const e=Q.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}R(Q,"close"),m(Q);const X='[data-bs-toggle="button"]';class Y extends W{static get NAME(){return"button"}toggle(){this._element.setAttribute("aria-pressed",this._element.classList.toggle("active"))}static jQueryInterface(t){return this.each((function(){const e=Y.getOrCreateInstance(this);"toggle"===t&&e[t]()}))}}N.on(document,"click.bs.button.data-api",X,(t=>{t.preventDefault();const e=t.target.closest(X);Y.getOrCreateInstance(e).toggle()})),m(Y);const U=".bs.swipe",G=`touchstart${U}`,J=`touchmove${U}`,Z=`touchend${U}`,tt=`pointerdown${U}`,et=`pointerup${U}`,it={endCallback:null,leftCallback:null,rightCallback:null},nt={endCallback:"(function|null)",leftCallback:"(function|null)",rightCallback:"(function|null)"};class st extends H{constructor(t,e){super(),this._element=t,t&&st.isSupported()&&(this._config=this._getConfig(e),this._deltaX=0,this._supportPointerEvents=Boolean(window.PointerEvent),this._initEvents())}static get Default(){return it}static get DefaultType(){return nt}static get NAME(){return"swipe"}dispose(){N.off(this._element,U)}_start(t){this._supportPointerEvents?this._eventIsPointerPenTouch(t)&&(this._deltaX=t.clientX):this._deltaX=t.touches[0].clientX}_end(t){this._eventIsPointerPenTouch(t)&&(this._deltaX=t.clientX-this._deltaX),this._handleSwipe(),g(this._config.endCallback)}_move(t){this._deltaX=t.touches&&t.touches.length>1?0:t.touches[0].clientX-this._deltaX}_handleSwipe(){const t=Math.abs(this._deltaX);if(t<=40)return;const e=t/this._deltaX;this._deltaX=0,e&&g(e>0?this._config.rightCallback:this._config.leftCallback)}_initEvents(){this._supportPointerEvents?(N.on(this._element,tt,(t=>this._start(t))),N.on(this._element,et,(t=>this._end(t))),this._element.classList.add("pointer-event")):(N.on(this._element,G,(t=>this._start(t))),N.on(this._element,J,(t=>this._move(t))),N.on(this._element,Z,(t=>this._end(t))))}_eventIsPointerPenTouch(t){return this._supportPointerEvents&&("pen"===t.pointerType||"touch"===t.pointerType)}static isSupported(){return"ontouchstart"in document.documentElement||navigator.maxTouchPoints>0}}const ot=".bs.carousel",rt=".data-api",at="next",lt="prev",ct="left",ht="right",dt=`slide${ot}`,ut=`slid${ot}`,ft=`keydown${ot}`,pt=`mouseenter${ot}`,mt=`mouseleave${ot}`,gt=`dragstart${ot}`,_t=`load${ot}${rt}`,bt=`click${ot}${rt}`,vt="carousel",yt="active",wt=".active",At=".carousel-item",Et=wt+At,Tt={ArrowLeft:ht,ArrowRight:ct},Ct={interval:5e3,keyboard:!0,pause:"hover",ride:!1,touch:!0,wrap:!0},Ot={interval:"(number|boolean)",keyboard:"boolean",pause:"(string|boolean)",ride:"(boolean|string)",touch:"boolean",wrap:"boolean"};class xt extends W{constructor(t,e){super(t,e),this._interval=null,this._activeElement=null,this._isSliding=!1,this.touchTimeout=null,this._swipeHelper=null,this._indicatorsElement=z.findOne(".carousel-indicators",this._element),this._addEventListeners(),this._config.ride===vt&&this.cycle()}static get Default(){return Ct}static get DefaultType(){return Ot}static get NAME(){return"carousel"}next(){this._slide(at)}nextWhenVisible(){!document.hidden&&a(this._element)&&this.next()}prev(){this._slide(lt)}pause(){this._isSliding&&s(this._element),this._clearInterval()}cycle(){this._clearInterval(),this._updateInterval(),this._interval=setInterval((()=>this.nextWhenVisible()),this._config.interval)}_maybeEnableCycle(){this._config.ride&&(this._isSliding?N.one(this._element,ut,(()=>this.cycle())):this.cycle())}to(t){const e=this._getItems();if(t>e.length-1||t<0)return;if(this._isSliding)return void N.one(this._element,ut,(()=>this.to(t)));const i=this._getItemIndex(this._getActive());if(i===t)return;const n=t>i?at:lt;this._slide(n,e[t])}dispose(){this._swipeHelper&&this._swipeHelper.dispose(),super.dispose()}_configAfterMerge(t){return t.defaultInterval=t.interval,t}_addEventListeners(){this._config.keyboard&&N.on(this._element,ft,(t=>this._keydown(t))),"hover"===this._config.pause&&(N.on(this._element,pt,(()=>this.pause())),N.on(this._element,mt,(()=>this._maybeEnableCycle()))),this._config.touch&&st.isSupported()&&this._addTouchEventListeners()}_addTouchEventListeners(){for(const t of z.find(".carousel-item img",this._element))N.on(t,gt,(t=>t.preventDefault()));const t={leftCallback:()=>this._slide(this._directionToOrder(ct)),rightCallback:()=>this._slide(this._directionToOrder(ht)),endCallback:()=>{"hover"===this._config.pause&&(this.pause(),this.touchTimeout&&clearTimeout(this.touchTimeout),this.touchTimeout=setTimeout((()=>this._maybeEnableCycle()),500+this._config.interval))}};this._swipeHelper=new st(this._element,t)}_keydown(t){if(/input|textarea/i.test(t.target.tagName))return;const e=Tt[t.key];e&&(t.preventDefault(),this._slide(this._directionToOrder(e)))}_getItemIndex(t){return this._getItems().indexOf(t)}_setActiveIndicatorElement(t){if(!this._indicatorsElement)return;const e=z.findOne(wt,this._indicatorsElement);e.classList.remove(yt),e.removeAttribute("aria-current");const i=z.findOne(`[data-bs-slide-to="${t}"]`,this._indicatorsElement);i&&(i.classList.add(yt),i.setAttribute("aria-current","true"))}_updateInterval(){const t=this._activeElement||this._getActive();if(!t)return;const e=Number.parseInt(t.getAttribute("data-bs-interval"),10);this._config.interval=e||this._config.defaultInterval}_slide(t,e=null){if(this._isSliding)return;const i=this._getActive(),n=t===at,s=e||b(this._getItems(),i,n,this._config.wrap);if(s===i)return;const o=this._getItemIndex(s),r=e=>N.trigger(this._element,e,{relatedTarget:s,direction:this._orderToDirection(t),from:this._getItemIndex(i),to:o});if(r(dt).defaultPrevented)return;if(!i||!s)return;const a=Boolean(this._interval);this.pause(),this._isSliding=!0,this._setActiveIndicatorElement(o),this._activeElement=s;const l=n?"carousel-item-start":"carousel-item-end",c=n?"carousel-item-next":"carousel-item-prev";s.classList.add(c),d(s),i.classList.add(l),s.classList.add(l),this._queueCallback((()=>{s.classList.remove(l,c),s.classList.add(yt),i.classList.remove(yt,c,l),this._isSliding=!1,r(ut)}),i,this._isAnimated()),a&&this.cycle()}_isAnimated(){return this._element.classList.contains("slide")}_getActive(){return z.findOne(Et,this._element)}_getItems(){return z.find(At,this._element)}_clearInterval(){this._interval&&(clearInterval(this._interval),this._interval=null)}_directionToOrder(t){return p()?t===ct?lt:at:t===ct?at:lt}_orderToDirection(t){return p()?t===lt?ct:ht:t===lt?ht:ct}static jQueryInterface(t){return this.each((function(){const e=xt.getOrCreateInstance(this,t);if("number"!=typeof t){if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}else e.to(t)}))}}N.on(document,bt,"[data-bs-slide], [data-bs-slide-to]",(function(t){const e=z.getElementFromSelector(this);if(!e||!e.classList.contains(vt))return;t.preventDefault();const i=xt.getOrCreateInstance(e),n=this.getAttribute("data-bs-slide-to");return n?(i.to(n),void i._maybeEnableCycle()):"next"===F.getDataAttribute(this,"slide")?(i.next(),void i._maybeEnableCycle()):(i.prev(),void i._maybeEnableCycle())})),N.on(window,_t,(()=>{const t=z.find('[data-bs-ride="carousel"]');for(const e of t)xt.getOrCreateInstance(e)})),m(xt);const kt=".bs.collapse",Lt=`show${kt}`,St=`shown${kt}`,Dt=`hide${kt}`,$t=`hidden${kt}`,It=`click${kt}.data-api`,Nt="show",Pt="collapse",Mt="collapsing",jt=`:scope .${Pt} .${Pt}`,Ft='[data-bs-toggle="collapse"]',Ht={parent:null,toggle:!0},Wt={parent:"(null|element)",toggle:"boolean"};class Bt extends W{constructor(t,e){super(t,e),this._isTransitioning=!1,this._triggerArray=[];const i=z.find(Ft);for(const t of i){const e=z.getSelectorFromElement(t),i=z.find(e).filter((t=>t===this._element));null!==e&&i.length&&this._triggerArray.push(t)}this._initializeChildren(),this._config.parent||this._addAriaAndCollapsedClass(this._triggerArray,this._isShown()),this._config.toggle&&this.toggle()}static get Default(){return Ht}static get DefaultType(){return Wt}static get NAME(){return"collapse"}toggle(){this._isShown()?this.hide():this.show()}show(){if(this._isTransitioning||this._isShown())return;let t=[];if(this._config.parent&&(t=this._getFirstLevelChildren(".collapse.show, .collapse.collapsing").filter((t=>t!==this._element)).map((t=>Bt.getOrCreateInstance(t,{toggle:!1})))),t.length&&t[0]._isTransitioning)return;if(N.trigger(this._element,Lt).defaultPrevented)return;for(const e of t)e.hide();const e=this._getDimension();this._element.classList.remove(Pt),this._element.classList.add(Mt),this._element.style[e]=0,this._addAriaAndCollapsedClass(this._triggerArray,!0),this._isTransitioning=!0;const i=`scroll${e[0].toUpperCase()+e.slice(1)}`;this._queueCallback((()=>{this._isTransitioning=!1,this._element.classList.remove(Mt),this._element.classList.add(Pt,Nt),this._element.style[e]="",N.trigger(this._element,St)}),this._element,!0),this._element.style[e]=`${this._element[i]}px`}hide(){if(this._isTransitioning||!this._isShown())return;if(N.trigger(this._element,Dt).defaultPrevented)return;const t=this._getDimension();this._element.style[t]=`${this._element.getBoundingClientRect()[t]}px`,d(this._element),this._element.classList.add(Mt),this._element.classList.remove(Pt,Nt);for(const t of this._triggerArray){const e=z.getElementFromSelector(t);e&&!this._isShown(e)&&this._addAriaAndCollapsedClass([t],!1)}this._isTransitioning=!0,this._element.style[t]="",this._queueCallback((()=>{this._isTransitioning=!1,this._element.classList.remove(Mt),this._element.classList.add(Pt),N.trigger(this._element,$t)}),this._element,!0)}_isShown(t=this._element){return t.classList.contains(Nt)}_configAfterMerge(t){return t.toggle=Boolean(t.toggle),t.parent=r(t.parent),t}_getDimension(){return this._element.classList.contains("collapse-horizontal")?"width":"height"}_initializeChildren(){if(!this._config.parent)return;const t=this._getFirstLevelChildren(Ft);for(const e of t){const t=z.getElementFromSelector(e);t&&this._addAriaAndCollapsedClass([e],this._isShown(t))}}_getFirstLevelChildren(t){const e=z.find(jt,this._config.parent);return z.find(t,this._config.parent).filter((t=>!e.includes(t)))}_addAriaAndCollapsedClass(t,e){if(t.length)for(const i of t)i.classList.toggle("collapsed",!e),i.setAttribute("aria-expanded",e)}static jQueryInterface(t){const e={};return"string"==typeof t&&/show|hide/.test(t)&&(e.toggle=!1),this.each((function(){const i=Bt.getOrCreateInstance(this,e);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t]()}}))}}N.on(document,It,Ft,(function(t){("A"===t.target.tagName||t.delegateTarget&&"A"===t.delegateTarget.tagName)&&t.preventDefault();for(const t of z.getMultipleElementsFromSelector(this))Bt.getOrCreateInstance(t,{toggle:!1}).toggle()})),m(Bt);var zt="top",Rt="bottom",qt="right",Vt="left",Kt="auto",Qt=[zt,Rt,qt,Vt],Xt="start",Yt="end",Ut="clippingParents",Gt="viewport",Jt="popper",Zt="reference",te=Qt.reduce((function(t,e){return t.concat([e+"-"+Xt,e+"-"+Yt])}),[]),ee=[].concat(Qt,[Kt]).reduce((function(t,e){return t.concat([e,e+"-"+Xt,e+"-"+Yt])}),[]),ie="beforeRead",ne="read",se="afterRead",oe="beforeMain",re="main",ae="afterMain",le="beforeWrite",ce="write",he="afterWrite",de=[ie,ne,se,oe,re,ae,le,ce,he];function ue(t){return t?(t.nodeName||"").toLowerCase():null}function fe(t){if(null==t)return window;if("[object Window]"!==t.toString()){var e=t.ownerDocument;return e&&e.defaultView||window}return t}function pe(t){return t instanceof fe(t).Element||t instanceof Element}function me(t){return t instanceof fe(t).HTMLElement||t instanceof HTMLElement}function ge(t){return"undefined"!=typeof ShadowRoot&&(t instanceof fe(t).ShadowRoot||t instanceof ShadowRoot)}const _e={name:"applyStyles",enabled:!0,phase:"write",fn:function(t){var e=t.state;Object.keys(e.elements).forEach((function(t){var i=e.styles[t]||{},n=e.attributes[t]||{},s=e.elements[t];me(s)&&ue(s)&&(Object.assign(s.style,i),Object.keys(n).forEach((function(t){var e=n[t];!1===e?s.removeAttribute(t):s.setAttribute(t,!0===e?"":e)})))}))},effect:function(t){var e=t.state,i={popper:{position:e.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(e.elements.popper.style,i.popper),e.styles=i,e.elements.arrow&&Object.assign(e.elements.arrow.style,i.arrow),function(){Object.keys(e.elements).forEach((function(t){var n=e.elements[t],s=e.attributes[t]||{},o=Object.keys(e.styles.hasOwnProperty(t)?e.styles[t]:i[t]).reduce((function(t,e){return t[e]="",t}),{});me(n)&&ue(n)&&(Object.assign(n.style,o),Object.keys(s).forEach((function(t){n.removeAttribute(t)})))}))}},requires:["computeStyles"]};function be(t){return t.split("-")[0]}var ve=Math.max,ye=Math.min,we=Math.round;function Ae(){var t=navigator.userAgentData;return null!=t&&t.brands&&Array.isArray(t.brands)?t.brands.map((function(t){return t.brand+"/"+t.version})).join(" "):navigator.userAgent}function Ee(){return!/^((?!chrome|android).)*safari/i.test(Ae())}function Te(t,e,i){void 0===e&&(e=!1),void 0===i&&(i=!1);var n=t.getBoundingClientRect(),s=1,o=1;e&&me(t)&&(s=t.offsetWidth>0&&we(n.width)/t.offsetWidth||1,o=t.offsetHeight>0&&we(n.height)/t.offsetHeight||1);var r=(pe(t)?fe(t):window).visualViewport,a=!Ee()&&i,l=(n.left+(a&&r?r.offsetLeft:0))/s,c=(n.top+(a&&r?r.offsetTop:0))/o,h=n.width/s,d=n.height/o;return{width:h,height:d,top:c,right:l+h,bottom:c+d,left:l,x:l,y:c}}function Ce(t){var e=Te(t),i=t.offsetWidth,n=t.offsetHeight;return Math.abs(e.width-i)<=1&&(i=e.width),Math.abs(e.height-n)<=1&&(n=e.height),{x:t.offsetLeft,y:t.offsetTop,width:i,height:n}}function Oe(t,e){var i=e.getRootNode&&e.getRootNode();if(t.contains(e))return!0;if(i&&ge(i)){var n=e;do{if(n&&t.isSameNode(n))return!0;n=n.parentNode||n.host}while(n)}return!1}function xe(t){return fe(t).getComputedStyle(t)}function ke(t){return["table","td","th"].indexOf(ue(t))>=0}function Le(t){return((pe(t)?t.ownerDocument:t.document)||window.document).documentElement}function Se(t){return"html"===ue(t)?t:t.assignedSlot||t.parentNode||(ge(t)?t.host:null)||Le(t)}function De(t){return me(t)&&"fixed"!==xe(t).position?t.offsetParent:null}function $e(t){for(var e=fe(t),i=De(t);i&&ke(i)&&"static"===xe(i).position;)i=De(i);return i&&("html"===ue(i)||"body"===ue(i)&&"static"===xe(i).position)?e:i||function(t){var e=/firefox/i.test(Ae());if(/Trident/i.test(Ae())&&me(t)&&"fixed"===xe(t).position)return null;var i=Se(t);for(ge(i)&&(i=i.host);me(i)&&["html","body"].indexOf(ue(i))<0;){var n=xe(i);if("none"!==n.transform||"none"!==n.perspective||"paint"===n.contain||-1!==["transform","perspective"].indexOf(n.willChange)||e&&"filter"===n.willChange||e&&n.filter&&"none"!==n.filter)return i;i=i.parentNode}return null}(t)||e}function Ie(t){return["top","bottom"].indexOf(t)>=0?"x":"y"}function Ne(t,e,i){return ve(t,ye(e,i))}function Pe(t){return Object.assign({},{top:0,right:0,bottom:0,left:0},t)}function Me(t,e){return e.reduce((function(e,i){return e[i]=t,e}),{})}const je={name:"arrow",enabled:!0,phase:"main",fn:function(t){var e,i=t.state,n=t.name,s=t.options,o=i.elements.arrow,r=i.modifiersData.popperOffsets,a=be(i.placement),l=Ie(a),c=[Vt,qt].indexOf(a)>=0?"height":"width";if(o&&r){var h=function(t,e){return Pe("number"!=typeof(t="function"==typeof t?t(Object.assign({},e.rects,{placement:e.placement})):t)?t:Me(t,Qt))}(s.padding,i),d=Ce(o),u="y"===l?zt:Vt,f="y"===l?Rt:qt,p=i.rects.reference[c]+i.rects.reference[l]-r[l]-i.rects.popper[c],m=r[l]-i.rects.reference[l],g=$e(o),_=g?"y"===l?g.clientHeight||0:g.clientWidth||0:0,b=p/2-m/2,v=h[u],y=_-d[c]-h[f],w=_/2-d[c]/2+b,A=Ne(v,w,y),E=l;i.modifiersData[n]=((e={})[E]=A,e.centerOffset=A-w,e)}},effect:function(t){var e=t.state,i=t.options.element,n=void 0===i?"[data-popper-arrow]":i;null!=n&&("string"!=typeof n||(n=e.elements.popper.querySelector(n)))&&Oe(e.elements.popper,n)&&(e.elements.arrow=n)},requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};function Fe(t){return t.split("-")[1]}var He={top:"auto",right:"auto",bottom:"auto",left:"auto"};function We(t){var e,i=t.popper,n=t.popperRect,s=t.placement,o=t.variation,r=t.offsets,a=t.position,l=t.gpuAcceleration,c=t.adaptive,h=t.roundOffsets,d=t.isFixed,u=r.x,f=void 0===u?0:u,p=r.y,m=void 0===p?0:p,g="function"==typeof h?h({x:f,y:m}):{x:f,y:m};f=g.x,m=g.y;var _=r.hasOwnProperty("x"),b=r.hasOwnProperty("y"),v=Vt,y=zt,w=window;if(c){var A=$e(i),E="clientHeight",T="clientWidth";A===fe(i)&&"static"!==xe(A=Le(i)).position&&"absolute"===a&&(E="scrollHeight",T="scrollWidth"),(s===zt||(s===Vt||s===qt)&&o===Yt)&&(y=Rt,m-=(d&&A===w&&w.visualViewport?w.visualViewport.height:A[E])-n.height,m*=l?1:-1),s!==Vt&&(s!==zt&&s!==Rt||o!==Yt)||(v=qt,f-=(d&&A===w&&w.visualViewport?w.visualViewport.width:A[T])-n.width,f*=l?1:-1)}var C,O=Object.assign({position:a},c&&He),x=!0===h?function(t,e){var i=t.x,n=t.y,s=e.devicePixelRatio||1;return{x:we(i*s)/s||0,y:we(n*s)/s||0}}({x:f,y:m},fe(i)):{x:f,y:m};return f=x.x,m=x.y,l?Object.assign({},O,((C={})[y]=b?"0":"",C[v]=_?"0":"",C.transform=(w.devicePixelRatio||1)<=1?"translate("+f+"px, "+m+"px)":"translate3d("+f+"px, "+m+"px, 0)",C)):Object.assign({},O,((e={})[y]=b?m+"px":"",e[v]=_?f+"px":"",e.transform="",e))}const Be={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:function(t){var e=t.state,i=t.options,n=i.gpuAcceleration,s=void 0===n||n,o=i.adaptive,r=void 0===o||o,a=i.roundOffsets,l=void 0===a||a,c={placement:be(e.placement),variation:Fe(e.placement),popper:e.elements.popper,popperRect:e.rects.popper,gpuAcceleration:s,isFixed:"fixed"===e.options.strategy};null!=e.modifiersData.popperOffsets&&(e.styles.popper=Object.assign({},e.styles.popper,We(Object.assign({},c,{offsets:e.modifiersData.popperOffsets,position:e.options.strategy,adaptive:r,roundOffsets:l})))),null!=e.modifiersData.arrow&&(e.styles.arrow=Object.assign({},e.styles.arrow,We(Object.assign({},c,{offsets:e.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:l})))),e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-placement":e.placement})},data:{}};var ze={passive:!0};const Re={name:"eventListeners",enabled:!0,phase:"write",fn:function(){},effect:function(t){var e=t.state,i=t.instance,n=t.options,s=n.scroll,o=void 0===s||s,r=n.resize,a=void 0===r||r,l=fe(e.elements.popper),c=[].concat(e.scrollParents.reference,e.scrollParents.popper);return o&&c.forEach((function(t){t.addEventListener("scroll",i.update,ze)})),a&&l.addEventListener("resize",i.update,ze),function(){o&&c.forEach((function(t){t.removeEventListener("scroll",i.update,ze)})),a&&l.removeEventListener("resize",i.update,ze)}},data:{}};var qe={left:"right",right:"left",bottom:"top",top:"bottom"};function Ve(t){return t.replace(/left|right|bottom|top/g,(function(t){return qe[t]}))}var Ke={start:"end",end:"start"};function Qe(t){return t.replace(/start|end/g,(function(t){return Ke[t]}))}function Xe(t){var e=fe(t);return{scrollLeft:e.pageXOffset,scrollTop:e.pageYOffset}}function Ye(t){return Te(Le(t)).left+Xe(t).scrollLeft}function Ue(t){var e=xe(t),i=e.overflow,n=e.overflowX,s=e.overflowY;return/auto|scroll|overlay|hidden/.test(i+s+n)}function Ge(t){return["html","body","#document"].indexOf(ue(t))>=0?t.ownerDocument.body:me(t)&&Ue(t)?t:Ge(Se(t))}function Je(t,e){var i;void 0===e&&(e=[]);var n=Ge(t),s=n===(null==(i=t.ownerDocument)?void 0:i.body),o=fe(n),r=s?[o].concat(o.visualViewport||[],Ue(n)?n:[]):n,a=e.concat(r);return s?a:a.concat(Je(Se(r)))}function Ze(t){return Object.assign({},t,{left:t.x,top:t.y,right:t.x+t.width,bottom:t.y+t.height})}function ti(t,e,i){return e===Gt?Ze(function(t,e){var i=fe(t),n=Le(t),s=i.visualViewport,o=n.clientWidth,r=n.clientHeight,a=0,l=0;if(s){o=s.width,r=s.height;var c=Ee();(c||!c&&"fixed"===e)&&(a=s.offsetLeft,l=s.offsetTop)}return{width:o,height:r,x:a+Ye(t),y:l}}(t,i)):pe(e)?function(t,e){var i=Te(t,!1,"fixed"===e);return i.top=i.top+t.clientTop,i.left=i.left+t.clientLeft,i.bottom=i.top+t.clientHeight,i.right=i.left+t.clientWidth,i.width=t.clientWidth,i.height=t.clientHeight,i.x=i.left,i.y=i.top,i}(e,i):Ze(function(t){var e,i=Le(t),n=Xe(t),s=null==(e=t.ownerDocument)?void 0:e.body,o=ve(i.scrollWidth,i.clientWidth,s?s.scrollWidth:0,s?s.clientWidth:0),r=ve(i.scrollHeight,i.clientHeight,s?s.scrollHeight:0,s?s.clientHeight:0),a=-n.scrollLeft+Ye(t),l=-n.scrollTop;return"rtl"===xe(s||i).direction&&(a+=ve(i.clientWidth,s?s.clientWidth:0)-o),{width:o,height:r,x:a,y:l}}(Le(t)))}function ei(t){var e,i=t.reference,n=t.element,s=t.placement,o=s?be(s):null,r=s?Fe(s):null,a=i.x+i.width/2-n.width/2,l=i.y+i.height/2-n.height/2;switch(o){case zt:e={x:a,y:i.y-n.height};break;case Rt:e={x:a,y:i.y+i.height};break;case qt:e={x:i.x+i.width,y:l};break;case Vt:e={x:i.x-n.width,y:l};break;default:e={x:i.x,y:i.y}}var c=o?Ie(o):null;if(null!=c){var h="y"===c?"height":"width";switch(r){case Xt:e[c]=e[c]-(i[h]/2-n[h]/2);break;case Yt:e[c]=e[c]+(i[h]/2-n[h]/2)}}return e}function ii(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=void 0===n?t.placement:n,o=i.strategy,r=void 0===o?t.strategy:o,a=i.boundary,l=void 0===a?Ut:a,c=i.rootBoundary,h=void 0===c?Gt:c,d=i.elementContext,u=void 0===d?Jt:d,f=i.altBoundary,p=void 0!==f&&f,m=i.padding,g=void 0===m?0:m,_=Pe("number"!=typeof g?g:Me(g,Qt)),b=u===Jt?Zt:Jt,v=t.rects.popper,y=t.elements[p?b:u],w=function(t,e,i,n){var s="clippingParents"===e?function(t){var e=Je(Se(t)),i=["absolute","fixed"].indexOf(xe(t).position)>=0&&me(t)?$e(t):t;return pe(i)?e.filter((function(t){return pe(t)&&Oe(t,i)&&"body"!==ue(t)})):[]}(t):[].concat(e),o=[].concat(s,[i]),r=o[0],a=o.reduce((function(e,i){var s=ti(t,i,n);return e.top=ve(s.top,e.top),e.right=ye(s.right,e.right),e.bottom=ye(s.bottom,e.bottom),e.left=ve(s.left,e.left),e}),ti(t,r,n));return a.width=a.right-a.left,a.height=a.bottom-a.top,a.x=a.left,a.y=a.top,a}(pe(y)?y:y.contextElement||Le(t.elements.popper),l,h,r),A=Te(t.elements.reference),E=ei({reference:A,element:v,strategy:"absolute",placement:s}),T=Ze(Object.assign({},v,E)),C=u===Jt?T:A,O={top:w.top-C.top+_.top,bottom:C.bottom-w.bottom+_.bottom,left:w.left-C.left+_.left,right:C.right-w.right+_.right},x=t.modifiersData.offset;if(u===Jt&&x){var k=x[s];Object.keys(O).forEach((function(t){var e=[qt,Rt].indexOf(t)>=0?1:-1,i=[zt,Rt].indexOf(t)>=0?"y":"x";O[t]+=k[i]*e}))}return O}function ni(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=i.boundary,o=i.rootBoundary,r=i.padding,a=i.flipVariations,l=i.allowedAutoPlacements,c=void 0===l?ee:l,h=Fe(n),d=h?a?te:te.filter((function(t){return Fe(t)===h})):Qt,u=d.filter((function(t){return c.indexOf(t)>=0}));0===u.length&&(u=d);var f=u.reduce((function(e,i){return e[i]=ii(t,{placement:i,boundary:s,rootBoundary:o,padding:r})[be(i)],e}),{});return Object.keys(f).sort((function(t,e){return f[t]-f[e]}))}const si={name:"flip",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name;if(!e.modifiersData[n]._skip){for(var s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0===r||r,l=i.fallbackPlacements,c=i.padding,h=i.boundary,d=i.rootBoundary,u=i.altBoundary,f=i.flipVariations,p=void 0===f||f,m=i.allowedAutoPlacements,g=e.options.placement,_=be(g),b=l||(_!==g&&p?function(t){if(be(t)===Kt)return[];var e=Ve(t);return[Qe(t),e,Qe(e)]}(g):[Ve(g)]),v=[g].concat(b).reduce((function(t,i){return t.concat(be(i)===Kt?ni(e,{placement:i,boundary:h,rootBoundary:d,padding:c,flipVariations:p,allowedAutoPlacements:m}):i)}),[]),y=e.rects.reference,w=e.rects.popper,A=new Map,E=!0,T=v[0],C=0;C=0,S=L?"width":"height",D=ii(e,{placement:O,boundary:h,rootBoundary:d,altBoundary:u,padding:c}),$=L?k?qt:Vt:k?Rt:zt;y[S]>w[S]&&($=Ve($));var I=Ve($),N=[];if(o&&N.push(D[x]<=0),a&&N.push(D[$]<=0,D[I]<=0),N.every((function(t){return t}))){T=O,E=!1;break}A.set(O,N)}if(E)for(var P=function(t){var e=v.find((function(e){var i=A.get(e);if(i)return i.slice(0,t).every((function(t){return t}))}));if(e)return T=e,"break"},M=p?3:1;M>0&&"break"!==P(M);M--);e.placement!==T&&(e.modifiersData[n]._skip=!0,e.placement=T,e.reset=!0)}},requiresIfExists:["offset"],data:{_skip:!1}};function oi(t,e,i){return void 0===i&&(i={x:0,y:0}),{top:t.top-e.height-i.y,right:t.right-e.width+i.x,bottom:t.bottom-e.height+i.y,left:t.left-e.width-i.x}}function ri(t){return[zt,qt,Rt,Vt].some((function(e){return t[e]>=0}))}const ai={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:function(t){var e=t.state,i=t.name,n=e.rects.reference,s=e.rects.popper,o=e.modifiersData.preventOverflow,r=ii(e,{elementContext:"reference"}),a=ii(e,{altBoundary:!0}),l=oi(r,n),c=oi(a,s,o),h=ri(l),d=ri(c);e.modifiersData[i]={referenceClippingOffsets:l,popperEscapeOffsets:c,isReferenceHidden:h,hasPopperEscaped:d},e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-reference-hidden":h,"data-popper-escaped":d})}},li={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.offset,o=void 0===s?[0,0]:s,r=ee.reduce((function(t,i){return t[i]=function(t,e,i){var n=be(t),s=[Vt,zt].indexOf(n)>=0?-1:1,o="function"==typeof i?i(Object.assign({},e,{placement:t})):i,r=o[0],a=o[1];return r=r||0,a=(a||0)*s,[Vt,qt].indexOf(n)>=0?{x:a,y:r}:{x:r,y:a}}(i,e.rects,o),t}),{}),a=r[e.placement],l=a.x,c=a.y;null!=e.modifiersData.popperOffsets&&(e.modifiersData.popperOffsets.x+=l,e.modifiersData.popperOffsets.y+=c),e.modifiersData[n]=r}},ci={name:"popperOffsets",enabled:!0,phase:"read",fn:function(t){var e=t.state,i=t.name;e.modifiersData[i]=ei({reference:e.rects.reference,element:e.rects.popper,strategy:"absolute",placement:e.placement})},data:{}},hi={name:"preventOverflow",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0!==r&&r,l=i.boundary,c=i.rootBoundary,h=i.altBoundary,d=i.padding,u=i.tether,f=void 0===u||u,p=i.tetherOffset,m=void 0===p?0:p,g=ii(e,{boundary:l,rootBoundary:c,padding:d,altBoundary:h}),_=be(e.placement),b=Fe(e.placement),v=!b,y=Ie(_),w="x"===y?"y":"x",A=e.modifiersData.popperOffsets,E=e.rects.reference,T=e.rects.popper,C="function"==typeof m?m(Object.assign({},e.rects,{placement:e.placement})):m,O="number"==typeof C?{mainAxis:C,altAxis:C}:Object.assign({mainAxis:0,altAxis:0},C),x=e.modifiersData.offset?e.modifiersData.offset[e.placement]:null,k={x:0,y:0};if(A){if(o){var L,S="y"===y?zt:Vt,D="y"===y?Rt:qt,$="y"===y?"height":"width",I=A[y],N=I+g[S],P=I-g[D],M=f?-T[$]/2:0,j=b===Xt?E[$]:T[$],F=b===Xt?-T[$]:-E[$],H=e.elements.arrow,W=f&&H?Ce(H):{width:0,height:0},B=e.modifiersData["arrow#persistent"]?e.modifiersData["arrow#persistent"].padding:{top:0,right:0,bottom:0,left:0},z=B[S],R=B[D],q=Ne(0,E[$],W[$]),V=v?E[$]/2-M-q-z-O.mainAxis:j-q-z-O.mainAxis,K=v?-E[$]/2+M+q+R+O.mainAxis:F+q+R+O.mainAxis,Q=e.elements.arrow&&$e(e.elements.arrow),X=Q?"y"===y?Q.clientTop||0:Q.clientLeft||0:0,Y=null!=(L=null==x?void 0:x[y])?L:0,U=I+K-Y,G=Ne(f?ye(N,I+V-Y-X):N,I,f?ve(P,U):P);A[y]=G,k[y]=G-I}if(a){var J,Z="x"===y?zt:Vt,tt="x"===y?Rt:qt,et=A[w],it="y"===w?"height":"width",nt=et+g[Z],st=et-g[tt],ot=-1!==[zt,Vt].indexOf(_),rt=null!=(J=null==x?void 0:x[w])?J:0,at=ot?nt:et-E[it]-T[it]-rt+O.altAxis,lt=ot?et+E[it]+T[it]-rt-O.altAxis:st,ct=f&&ot?function(t,e,i){var n=Ne(t,e,i);return n>i?i:n}(at,et,lt):Ne(f?at:nt,et,f?lt:st);A[w]=ct,k[w]=ct-et}e.modifiersData[n]=k}},requiresIfExists:["offset"]};function di(t,e,i){void 0===i&&(i=!1);var n,s,o=me(e),r=me(e)&&function(t){var e=t.getBoundingClientRect(),i=we(e.width)/t.offsetWidth||1,n=we(e.height)/t.offsetHeight||1;return 1!==i||1!==n}(e),a=Le(e),l=Te(t,r,i),c={scrollLeft:0,scrollTop:0},h={x:0,y:0};return(o||!o&&!i)&&(("body"!==ue(e)||Ue(a))&&(c=(n=e)!==fe(n)&&me(n)?{scrollLeft:(s=n).scrollLeft,scrollTop:s.scrollTop}:Xe(n)),me(e)?((h=Te(e,!0)).x+=e.clientLeft,h.y+=e.clientTop):a&&(h.x=Ye(a))),{x:l.left+c.scrollLeft-h.x,y:l.top+c.scrollTop-h.y,width:l.width,height:l.height}}function ui(t){var e=new Map,i=new Set,n=[];function s(t){i.add(t.name),[].concat(t.requires||[],t.requiresIfExists||[]).forEach((function(t){if(!i.has(t)){var n=e.get(t);n&&s(n)}})),n.push(t)}return t.forEach((function(t){e.set(t.name,t)})),t.forEach((function(t){i.has(t.name)||s(t)})),n}var fi={placement:"bottom",modifiers:[],strategy:"absolute"};function pi(){for(var t=arguments.length,e=new Array(t),i=0;iNumber.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_getPopperConfig(){const t={placement:this._getPlacement(),modifiers:[{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"offset",options:{offset:this._getOffset()}}]};return(this._inNavbar||"static"===this._config.display)&&(F.setDataAttribute(this._menu,"popper","static"),t.modifiers=[{name:"applyStyles",enabled:!1}]),{...t,...g(this._config.popperConfig,[t])}}_selectMenuItem({key:t,target:e}){const i=z.find(".dropdown-menu .dropdown-item:not(.disabled):not(:disabled)",this._menu).filter((t=>a(t)));i.length&&b(i,e,t===Ti,!i.includes(e)).focus()}static jQueryInterface(t){return this.each((function(){const e=qi.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}static clearMenus(t){if(2===t.button||"keyup"===t.type&&"Tab"!==t.key)return;const e=z.find(Ni);for(const i of e){const e=qi.getInstance(i);if(!e||!1===e._config.autoClose)continue;const n=t.composedPath(),s=n.includes(e._menu);if(n.includes(e._element)||"inside"===e._config.autoClose&&!s||"outside"===e._config.autoClose&&s)continue;if(e._menu.contains(t.target)&&("keyup"===t.type&&"Tab"===t.key||/input|select|option|textarea|form/i.test(t.target.tagName)))continue;const o={relatedTarget:e._element};"click"===t.type&&(o.clickEvent=t),e._completeHide(o)}}static dataApiKeydownHandler(t){const e=/input|textarea/i.test(t.target.tagName),i="Escape"===t.key,n=[Ei,Ti].includes(t.key);if(!n&&!i)return;if(e&&!i)return;t.preventDefault();const s=this.matches(Ii)?this:z.prev(this,Ii)[0]||z.next(this,Ii)[0]||z.findOne(Ii,t.delegateTarget.parentNode),o=qi.getOrCreateInstance(s);if(n)return t.stopPropagation(),o.show(),void o._selectMenuItem(t);o._isShown()&&(t.stopPropagation(),o.hide(),s.focus())}}N.on(document,Si,Ii,qi.dataApiKeydownHandler),N.on(document,Si,Pi,qi.dataApiKeydownHandler),N.on(document,Li,qi.clearMenus),N.on(document,Di,qi.clearMenus),N.on(document,Li,Ii,(function(t){t.preventDefault(),qi.getOrCreateInstance(this).toggle()})),m(qi);const Vi="backdrop",Ki="show",Qi=`mousedown.bs.${Vi}`,Xi={className:"modal-backdrop",clickCallback:null,isAnimated:!1,isVisible:!0,rootElement:"body"},Yi={className:"string",clickCallback:"(function|null)",isAnimated:"boolean",isVisible:"boolean",rootElement:"(element|string)"};class Ui extends H{constructor(t){super(),this._config=this._getConfig(t),this._isAppended=!1,this._element=null}static get Default(){return Xi}static get DefaultType(){return Yi}static get NAME(){return Vi}show(t){if(!this._config.isVisible)return void g(t);this._append();const e=this._getElement();this._config.isAnimated&&d(e),e.classList.add(Ki),this._emulateAnimation((()=>{g(t)}))}hide(t){this._config.isVisible?(this._getElement().classList.remove(Ki),this._emulateAnimation((()=>{this.dispose(),g(t)}))):g(t)}dispose(){this._isAppended&&(N.off(this._element,Qi),this._element.remove(),this._isAppended=!1)}_getElement(){if(!this._element){const t=document.createElement("div");t.className=this._config.className,this._config.isAnimated&&t.classList.add("fade"),this._element=t}return this._element}_configAfterMerge(t){return t.rootElement=r(t.rootElement),t}_append(){if(this._isAppended)return;const t=this._getElement();this._config.rootElement.append(t),N.on(t,Qi,(()=>{g(this._config.clickCallback)})),this._isAppended=!0}_emulateAnimation(t){_(t,this._getElement(),this._config.isAnimated)}}const Gi=".bs.focustrap",Ji=`focusin${Gi}`,Zi=`keydown.tab${Gi}`,tn="backward",en={autofocus:!0,trapElement:null},nn={autofocus:"boolean",trapElement:"element"};class sn extends H{constructor(t){super(),this._config=this._getConfig(t),this._isActive=!1,this._lastTabNavDirection=null}static get Default(){return en}static get DefaultType(){return nn}static get NAME(){return"focustrap"}activate(){this._isActive||(this._config.autofocus&&this._config.trapElement.focus(),N.off(document,Gi),N.on(document,Ji,(t=>this._handleFocusin(t))),N.on(document,Zi,(t=>this._handleKeydown(t))),this._isActive=!0)}deactivate(){this._isActive&&(this._isActive=!1,N.off(document,Gi))}_handleFocusin(t){const{trapElement:e}=this._config;if(t.target===document||t.target===e||e.contains(t.target))return;const i=z.focusableChildren(e);0===i.length?e.focus():this._lastTabNavDirection===tn?i[i.length-1].focus():i[0].focus()}_handleKeydown(t){"Tab"===t.key&&(this._lastTabNavDirection=t.shiftKey?tn:"forward")}}const on=".fixed-top, .fixed-bottom, .is-fixed, .sticky-top",rn=".sticky-top",an="padding-right",ln="margin-right";class cn{constructor(){this._element=document.body}getWidth(){const t=document.documentElement.clientWidth;return Math.abs(window.innerWidth-t)}hide(){const t=this.getWidth();this._disableOverFlow(),this._setElementAttributes(this._element,an,(e=>e+t)),this._setElementAttributes(on,an,(e=>e+t)),this._setElementAttributes(rn,ln,(e=>e-t))}reset(){this._resetElementAttributes(this._element,"overflow"),this._resetElementAttributes(this._element,an),this._resetElementAttributes(on,an),this._resetElementAttributes(rn,ln)}isOverflowing(){return this.getWidth()>0}_disableOverFlow(){this._saveInitialAttribute(this._element,"overflow"),this._element.style.overflow="hidden"}_setElementAttributes(t,e,i){const n=this.getWidth();this._applyManipulationCallback(t,(t=>{if(t!==this._element&&window.innerWidth>t.clientWidth+n)return;this._saveInitialAttribute(t,e);const s=window.getComputedStyle(t).getPropertyValue(e);t.style.setProperty(e,`${i(Number.parseFloat(s))}px`)}))}_saveInitialAttribute(t,e){const i=t.style.getPropertyValue(e);i&&F.setDataAttribute(t,e,i)}_resetElementAttributes(t,e){this._applyManipulationCallback(t,(t=>{const i=F.getDataAttribute(t,e);null!==i?(F.removeDataAttribute(t,e),t.style.setProperty(e,i)):t.style.removeProperty(e)}))}_applyManipulationCallback(t,e){if(o(t))e(t);else for(const i of z.find(t,this._element))e(i)}}const hn=".bs.modal",dn=`hide${hn}`,un=`hidePrevented${hn}`,fn=`hidden${hn}`,pn=`show${hn}`,mn=`shown${hn}`,gn=`resize${hn}`,_n=`click.dismiss${hn}`,bn=`mousedown.dismiss${hn}`,vn=`keydown.dismiss${hn}`,yn=`click${hn}.data-api`,wn="modal-open",An="show",En="modal-static",Tn={backdrop:!0,focus:!0,keyboard:!0},Cn={backdrop:"(boolean|string)",focus:"boolean",keyboard:"boolean"};class On extends W{constructor(t,e){super(t,e),this._dialog=z.findOne(".modal-dialog",this._element),this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._isShown=!1,this._isTransitioning=!1,this._scrollBar=new cn,this._addEventListeners()}static get Default(){return Tn}static get DefaultType(){return Cn}static get NAME(){return"modal"}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||this._isTransitioning||N.trigger(this._element,pn,{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._isTransitioning=!0,this._scrollBar.hide(),document.body.classList.add(wn),this._adjustDialog(),this._backdrop.show((()=>this._showElement(t))))}hide(){this._isShown&&!this._isTransitioning&&(N.trigger(this._element,dn).defaultPrevented||(this._isShown=!1,this._isTransitioning=!0,this._focustrap.deactivate(),this._element.classList.remove(An),this._queueCallback((()=>this._hideModal()),this._element,this._isAnimated())))}dispose(){N.off(window,hn),N.off(this._dialog,hn),this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}handleUpdate(){this._adjustDialog()}_initializeBackDrop(){return new Ui({isVisible:Boolean(this._config.backdrop),isAnimated:this._isAnimated()})}_initializeFocusTrap(){return new sn({trapElement:this._element})}_showElement(t){document.body.contains(this._element)||document.body.append(this._element),this._element.style.display="block",this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.scrollTop=0;const e=z.findOne(".modal-body",this._dialog);e&&(e.scrollTop=0),d(this._element),this._element.classList.add(An),this._queueCallback((()=>{this._config.focus&&this._focustrap.activate(),this._isTransitioning=!1,N.trigger(this._element,mn,{relatedTarget:t})}),this._dialog,this._isAnimated())}_addEventListeners(){N.on(this._element,vn,(t=>{"Escape"===t.key&&(this._config.keyboard?this.hide():this._triggerBackdropTransition())})),N.on(window,gn,(()=>{this._isShown&&!this._isTransitioning&&this._adjustDialog()})),N.on(this._element,bn,(t=>{N.one(this._element,_n,(e=>{this._element===t.target&&this._element===e.target&&("static"!==this._config.backdrop?this._config.backdrop&&this.hide():this._triggerBackdropTransition())}))}))}_hideModal(){this._element.style.display="none",this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._isTransitioning=!1,this._backdrop.hide((()=>{document.body.classList.remove(wn),this._resetAdjustments(),this._scrollBar.reset(),N.trigger(this._element,fn)}))}_isAnimated(){return this._element.classList.contains("fade")}_triggerBackdropTransition(){if(N.trigger(this._element,un).defaultPrevented)return;const t=this._element.scrollHeight>document.documentElement.clientHeight,e=this._element.style.overflowY;"hidden"===e||this._element.classList.contains(En)||(t||(this._element.style.overflowY="hidden"),this._element.classList.add(En),this._queueCallback((()=>{this._element.classList.remove(En),this._queueCallback((()=>{this._element.style.overflowY=e}),this._dialog)}),this._dialog),this._element.focus())}_adjustDialog(){const t=this._element.scrollHeight>document.documentElement.clientHeight,e=this._scrollBar.getWidth(),i=e>0;if(i&&!t){const t=p()?"paddingLeft":"paddingRight";this._element.style[t]=`${e}px`}if(!i&&t){const t=p()?"paddingRight":"paddingLeft";this._element.style[t]=`${e}px`}}_resetAdjustments(){this._element.style.paddingLeft="",this._element.style.paddingRight=""}static jQueryInterface(t,e){return this.each((function(){const i=On.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t](e)}}))}}N.on(document,yn,'[data-bs-toggle="modal"]',(function(t){const e=z.getElementFromSelector(this);["A","AREA"].includes(this.tagName)&&t.preventDefault(),N.one(e,pn,(t=>{t.defaultPrevented||N.one(e,fn,(()=>{a(this)&&this.focus()}))}));const i=z.findOne(".modal.show");i&&On.getInstance(i).hide(),On.getOrCreateInstance(e).toggle(this)})),R(On),m(On);const xn=".bs.offcanvas",kn=".data-api",Ln=`load${xn}${kn}`,Sn="show",Dn="showing",$n="hiding",In=".offcanvas.show",Nn=`show${xn}`,Pn=`shown${xn}`,Mn=`hide${xn}`,jn=`hidePrevented${xn}`,Fn=`hidden${xn}`,Hn=`resize${xn}`,Wn=`click${xn}${kn}`,Bn=`keydown.dismiss${xn}`,zn={backdrop:!0,keyboard:!0,scroll:!1},Rn={backdrop:"(boolean|string)",keyboard:"boolean",scroll:"boolean"};class qn extends W{constructor(t,e){super(t,e),this._isShown=!1,this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._addEventListeners()}static get Default(){return zn}static get DefaultType(){return Rn}static get NAME(){return"offcanvas"}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||N.trigger(this._element,Nn,{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._backdrop.show(),this._config.scroll||(new cn).hide(),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.classList.add(Dn),this._queueCallback((()=>{this._config.scroll&&!this._config.backdrop||this._focustrap.activate(),this._element.classList.add(Sn),this._element.classList.remove(Dn),N.trigger(this._element,Pn,{relatedTarget:t})}),this._element,!0))}hide(){this._isShown&&(N.trigger(this._element,Mn).defaultPrevented||(this._focustrap.deactivate(),this._element.blur(),this._isShown=!1,this._element.classList.add($n),this._backdrop.hide(),this._queueCallback((()=>{this._element.classList.remove(Sn,$n),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._config.scroll||(new cn).reset(),N.trigger(this._element,Fn)}),this._element,!0)))}dispose(){this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}_initializeBackDrop(){const t=Boolean(this._config.backdrop);return new Ui({className:"offcanvas-backdrop",isVisible:t,isAnimated:!0,rootElement:this._element.parentNode,clickCallback:t?()=>{"static"!==this._config.backdrop?this.hide():N.trigger(this._element,jn)}:null})}_initializeFocusTrap(){return new sn({trapElement:this._element})}_addEventListeners(){N.on(this._element,Bn,(t=>{"Escape"===t.key&&(this._config.keyboard?this.hide():N.trigger(this._element,jn))}))}static jQueryInterface(t){return this.each((function(){const e=qn.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}N.on(document,Wn,'[data-bs-toggle="offcanvas"]',(function(t){const e=z.getElementFromSelector(this);if(["A","AREA"].includes(this.tagName)&&t.preventDefault(),l(this))return;N.one(e,Fn,(()=>{a(this)&&this.focus()}));const i=z.findOne(In);i&&i!==e&&qn.getInstance(i).hide(),qn.getOrCreateInstance(e).toggle(this)})),N.on(window,Ln,(()=>{for(const t of z.find(In))qn.getOrCreateInstance(t).show()})),N.on(window,Hn,(()=>{for(const t of z.find("[aria-modal][class*=show][class*=offcanvas-]"))"fixed"!==getComputedStyle(t).position&&qn.getOrCreateInstance(t).hide()})),R(qn),m(qn);const Vn={"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","srcset","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},Kn=new Set(["background","cite","href","itemtype","longdesc","poster","src","xlink:href"]),Qn=/^(?!javascript:)(?:[a-z0-9+.-]+:|[^&:/?#]*(?:[/?#]|$))/i,Xn=(t,e)=>{const i=t.nodeName.toLowerCase();return e.includes(i)?!Kn.has(i)||Boolean(Qn.test(t.nodeValue)):e.filter((t=>t instanceof RegExp)).some((t=>t.test(i)))},Yn={allowList:Vn,content:{},extraClass:"",html:!1,sanitize:!0,sanitizeFn:null,template:"
"},Un={allowList:"object",content:"object",extraClass:"(string|function)",html:"boolean",sanitize:"boolean",sanitizeFn:"(null|function)",template:"string"},Gn={entry:"(string|element|function|null)",selector:"(string|element)"};class Jn extends H{constructor(t){super(),this._config=this._getConfig(t)}static get Default(){return Yn}static get DefaultType(){return Un}static get NAME(){return"TemplateFactory"}getContent(){return Object.values(this._config.content).map((t=>this._resolvePossibleFunction(t))).filter(Boolean)}hasContent(){return this.getContent().length>0}changeContent(t){return this._checkContent(t),this._config.content={...this._config.content,...t},this}toHtml(){const t=document.createElement("div");t.innerHTML=this._maybeSanitize(this._config.template);for(const[e,i]of Object.entries(this._config.content))this._setContent(t,i,e);const e=t.children[0],i=this._resolvePossibleFunction(this._config.extraClass);return i&&e.classList.add(...i.split(" ")),e}_typeCheckConfig(t){super._typeCheckConfig(t),this._checkContent(t.content)}_checkContent(t){for(const[e,i]of Object.entries(t))super._typeCheckConfig({selector:e,entry:i},Gn)}_setContent(t,e,i){const n=z.findOne(i,t);n&&((e=this._resolvePossibleFunction(e))?o(e)?this._putElementInTemplate(r(e),n):this._config.html?n.innerHTML=this._maybeSanitize(e):n.textContent=e:n.remove())}_maybeSanitize(t){return this._config.sanitize?function(t,e,i){if(!t.length)return t;if(i&&"function"==typeof i)return i(t);const n=(new window.DOMParser).parseFromString(t,"text/html"),s=[].concat(...n.body.querySelectorAll("*"));for(const t of s){const i=t.nodeName.toLowerCase();if(!Object.keys(e).includes(i)){t.remove();continue}const n=[].concat(...t.attributes),s=[].concat(e["*"]||[],e[i]||[]);for(const e of n)Xn(e,s)||t.removeAttribute(e.nodeName)}return n.body.innerHTML}(t,this._config.allowList,this._config.sanitizeFn):t}_resolvePossibleFunction(t){return g(t,[this])}_putElementInTemplate(t,e){if(this._config.html)return e.innerHTML="",void e.append(t);e.textContent=t.textContent}}const Zn=new Set(["sanitize","allowList","sanitizeFn"]),ts="fade",es="show",is=".modal",ns="hide.bs.modal",ss="hover",os="focus",rs={AUTO:"auto",TOP:"top",RIGHT:p()?"left":"right",BOTTOM:"bottom",LEFT:p()?"right":"left"},as={allowList:Vn,animation:!0,boundary:"clippingParents",container:!1,customClass:"",delay:0,fallbackPlacements:["top","right","bottom","left"],html:!1,offset:[0,6],placement:"top",popperConfig:null,sanitize:!0,sanitizeFn:null,selector:!1,template:'',title:"",trigger:"hover focus"},ls={allowList:"object",animation:"boolean",boundary:"(string|element)",container:"(string|element|boolean)",customClass:"(string|function)",delay:"(number|object)",fallbackPlacements:"array",html:"boolean",offset:"(array|string|function)",placement:"(string|function)",popperConfig:"(null|object|function)",sanitize:"boolean",sanitizeFn:"(null|function)",selector:"(string|boolean)",template:"string",title:"(string|element|function)",trigger:"string"};class cs extends W{constructor(t,e){if(void 0===vi)throw new TypeError("Bootstrap's tooltips require Popper (https://popper.js.org)");super(t,e),this._isEnabled=!0,this._timeout=0,this._isHovered=null,this._activeTrigger={},this._popper=null,this._templateFactory=null,this._newContent=null,this.tip=null,this._setListeners(),this._config.selector||this._fixTitle()}static get Default(){return as}static get DefaultType(){return ls}static get NAME(){return"tooltip"}enable(){this._isEnabled=!0}disable(){this._isEnabled=!1}toggleEnabled(){this._isEnabled=!this._isEnabled}toggle(){this._isEnabled&&(this._activeTrigger.click=!this._activeTrigger.click,this._isShown()?this._leave():this._enter())}dispose(){clearTimeout(this._timeout),N.off(this._element.closest(is),ns,this._hideModalHandler),this._element.getAttribute("data-bs-original-title")&&this._element.setAttribute("title",this._element.getAttribute("data-bs-original-title")),this._disposePopper(),super.dispose()}show(){if("none"===this._element.style.display)throw new Error("Please use show on visible elements");if(!this._isWithContent()||!this._isEnabled)return;const t=N.trigger(this._element,this.constructor.eventName("show")),e=(c(this._element)||this._element.ownerDocument.documentElement).contains(this._element);if(t.defaultPrevented||!e)return;this._disposePopper();const i=this._getTipElement();this._element.setAttribute("aria-describedby",i.getAttribute("id"));const{container:n}=this._config;if(this._element.ownerDocument.documentElement.contains(this.tip)||(n.append(i),N.trigger(this._element,this.constructor.eventName("inserted"))),this._popper=this._createPopper(i),i.classList.add(es),"ontouchstart"in document.documentElement)for(const t of[].concat(...document.body.children))N.on(t,"mouseover",h);this._queueCallback((()=>{N.trigger(this._element,this.constructor.eventName("shown")),!1===this._isHovered&&this._leave(),this._isHovered=!1}),this.tip,this._isAnimated())}hide(){if(this._isShown()&&!N.trigger(this._element,this.constructor.eventName("hide")).defaultPrevented){if(this._getTipElement().classList.remove(es),"ontouchstart"in document.documentElement)for(const t of[].concat(...document.body.children))N.off(t,"mouseover",h);this._activeTrigger.click=!1,this._activeTrigger[os]=!1,this._activeTrigger[ss]=!1,this._isHovered=null,this._queueCallback((()=>{this._isWithActiveTrigger()||(this._isHovered||this._disposePopper(),this._element.removeAttribute("aria-describedby"),N.trigger(this._element,this.constructor.eventName("hidden")))}),this.tip,this._isAnimated())}}update(){this._popper&&this._popper.update()}_isWithContent(){return Boolean(this._getTitle())}_getTipElement(){return this.tip||(this.tip=this._createTipElement(this._newContent||this._getContentForTemplate())),this.tip}_createTipElement(t){const e=this._getTemplateFactory(t).toHtml();if(!e)return null;e.classList.remove(ts,es),e.classList.add(`bs-${this.constructor.NAME}-auto`);const i=(t=>{do{t+=Math.floor(1e6*Math.random())}while(document.getElementById(t));return t})(this.constructor.NAME).toString();return e.setAttribute("id",i),this._isAnimated()&&e.classList.add(ts),e}setContent(t){this._newContent=t,this._isShown()&&(this._disposePopper(),this.show())}_getTemplateFactory(t){return this._templateFactory?this._templateFactory.changeContent(t):this._templateFactory=new Jn({...this._config,content:t,extraClass:this._resolvePossibleFunction(this._config.customClass)}),this._templateFactory}_getContentForTemplate(){return{".tooltip-inner":this._getTitle()}}_getTitle(){return this._resolvePossibleFunction(this._config.title)||this._element.getAttribute("data-bs-original-title")}_initializeOnDelegatedTarget(t){return this.constructor.getOrCreateInstance(t.delegateTarget,this._getDelegateConfig())}_isAnimated(){return this._config.animation||this.tip&&this.tip.classList.contains(ts)}_isShown(){return this.tip&&this.tip.classList.contains(es)}_createPopper(t){const e=g(this._config.placement,[this,t,this._element]),i=rs[e.toUpperCase()];return bi(this._element,t,this._getPopperConfig(i))}_getOffset(){const{offset:t}=this._config;return"string"==typeof t?t.split(",").map((t=>Number.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_resolvePossibleFunction(t){return g(t,[this._element])}_getPopperConfig(t){const e={placement:t,modifiers:[{name:"flip",options:{fallbackPlacements:this._config.fallbackPlacements}},{name:"offset",options:{offset:this._getOffset()}},{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"arrow",options:{element:`.${this.constructor.NAME}-arrow`}},{name:"preSetPlacement",enabled:!0,phase:"beforeMain",fn:t=>{this._getTipElement().setAttribute("data-popper-placement",t.state.placement)}}]};return{...e,...g(this._config.popperConfig,[e])}}_setListeners(){const t=this._config.trigger.split(" ");for(const e of t)if("click"===e)N.on(this._element,this.constructor.eventName("click"),this._config.selector,(t=>{this._initializeOnDelegatedTarget(t).toggle()}));else if("manual"!==e){const t=e===ss?this.constructor.eventName("mouseenter"):this.constructor.eventName("focusin"),i=e===ss?this.constructor.eventName("mouseleave"):this.constructor.eventName("focusout");N.on(this._element,t,this._config.selector,(t=>{const e=this._initializeOnDelegatedTarget(t);e._activeTrigger["focusin"===t.type?os:ss]=!0,e._enter()})),N.on(this._element,i,this._config.selector,(t=>{const e=this._initializeOnDelegatedTarget(t);e._activeTrigger["focusout"===t.type?os:ss]=e._element.contains(t.relatedTarget),e._leave()}))}this._hideModalHandler=()=>{this._element&&this.hide()},N.on(this._element.closest(is),ns,this._hideModalHandler)}_fixTitle(){const t=this._element.getAttribute("title");t&&(this._element.getAttribute("aria-label")||this._element.textContent.trim()||this._element.setAttribute("aria-label",t),this._element.setAttribute("data-bs-original-title",t),this._element.removeAttribute("title"))}_enter(){this._isShown()||this._isHovered?this._isHovered=!0:(this._isHovered=!0,this._setTimeout((()=>{this._isHovered&&this.show()}),this._config.delay.show))}_leave(){this._isWithActiveTrigger()||(this._isHovered=!1,this._setTimeout((()=>{this._isHovered||this.hide()}),this._config.delay.hide))}_setTimeout(t,e){clearTimeout(this._timeout),this._timeout=setTimeout(t,e)}_isWithActiveTrigger(){return Object.values(this._activeTrigger).includes(!0)}_getConfig(t){const e=F.getDataAttributes(this._element);for(const t of Object.keys(e))Zn.has(t)&&delete e[t];return t={...e,..."object"==typeof t&&t?t:{}},t=this._mergeConfigObj(t),t=this._configAfterMerge(t),this._typeCheckConfig(t),t}_configAfterMerge(t){return t.container=!1===t.container?document.body:r(t.container),"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),t}_getDelegateConfig(){const t={};for(const[e,i]of Object.entries(this._config))this.constructor.Default[e]!==i&&(t[e]=i);return t.selector=!1,t.trigger="manual",t}_disposePopper(){this._popper&&(this._popper.destroy(),this._popper=null),this.tip&&(this.tip.remove(),this.tip=null)}static jQueryInterface(t){return this.each((function(){const e=cs.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}m(cs);const hs={...cs.Default,content:"",offset:[0,8],placement:"right",template:'',trigger:"click"},ds={...cs.DefaultType,content:"(null|string|element|function)"};class us extends cs{static get Default(){return hs}static get DefaultType(){return ds}static get NAME(){return"popover"}_isWithContent(){return this._getTitle()||this._getContent()}_getContentForTemplate(){return{".popover-header":this._getTitle(),".popover-body":this._getContent()}}_getContent(){return this._resolvePossibleFunction(this._config.content)}static jQueryInterface(t){return this.each((function(){const e=us.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}m(us);const fs=".bs.scrollspy",ps=`activate${fs}`,ms=`click${fs}`,gs=`load${fs}.data-api`,_s="active",bs="[href]",vs=".nav-link",ys=`${vs}, .nav-item > ${vs}, .list-group-item`,ws={offset:null,rootMargin:"0px 0px -25%",smoothScroll:!1,target:null,threshold:[.1,.5,1]},As={offset:"(number|null)",rootMargin:"string",smoothScroll:"boolean",target:"element",threshold:"array"};class Es extends W{constructor(t,e){super(t,e),this._targetLinks=new Map,this._observableSections=new Map,this._rootElement="visible"===getComputedStyle(this._element).overflowY?null:this._element,this._activeTarget=null,this._observer=null,this._previousScrollData={visibleEntryTop:0,parentScrollTop:0},this.refresh()}static get Default(){return ws}static get DefaultType(){return As}static get NAME(){return"scrollspy"}refresh(){this._initializeTargetsAndObservables(),this._maybeEnableSmoothScroll(),this._observer?this._observer.disconnect():this._observer=this._getNewObserver();for(const t of this._observableSections.values())this._observer.observe(t)}dispose(){this._observer.disconnect(),super.dispose()}_configAfterMerge(t){return t.target=r(t.target)||document.body,t.rootMargin=t.offset?`${t.offset}px 0px -30%`:t.rootMargin,"string"==typeof t.threshold&&(t.threshold=t.threshold.split(",").map((t=>Number.parseFloat(t)))),t}_maybeEnableSmoothScroll(){this._config.smoothScroll&&(N.off(this._config.target,ms),N.on(this._config.target,ms,bs,(t=>{const e=this._observableSections.get(t.target.hash);if(e){t.preventDefault();const i=this._rootElement||window,n=e.offsetTop-this._element.offsetTop;if(i.scrollTo)return void i.scrollTo({top:n,behavior:"smooth"});i.scrollTop=n}})))}_getNewObserver(){const t={root:this._rootElement,threshold:this._config.threshold,rootMargin:this._config.rootMargin};return new IntersectionObserver((t=>this._observerCallback(t)),t)}_observerCallback(t){const e=t=>this._targetLinks.get(`#${t.target.id}`),i=t=>{this._previousScrollData.visibleEntryTop=t.target.offsetTop,this._process(e(t))},n=(this._rootElement||document.documentElement).scrollTop,s=n>=this._previousScrollData.parentScrollTop;this._previousScrollData.parentScrollTop=n;for(const o of t){if(!o.isIntersecting){this._activeTarget=null,this._clearActiveClass(e(o));continue}const t=o.target.offsetTop>=this._previousScrollData.visibleEntryTop;if(s&&t){if(i(o),!n)return}else s||t||i(o)}}_initializeTargetsAndObservables(){this._targetLinks=new Map,this._observableSections=new Map;const t=z.find(bs,this._config.target);for(const e of t){if(!e.hash||l(e))continue;const t=z.findOne(decodeURI(e.hash),this._element);a(t)&&(this._targetLinks.set(decodeURI(e.hash),e),this._observableSections.set(e.hash,t))}}_process(t){this._activeTarget!==t&&(this._clearActiveClass(this._config.target),this._activeTarget=t,t.classList.add(_s),this._activateParents(t),N.trigger(this._element,ps,{relatedTarget:t}))}_activateParents(t){if(t.classList.contains("dropdown-item"))z.findOne(".dropdown-toggle",t.closest(".dropdown")).classList.add(_s);else for(const e of z.parents(t,".nav, .list-group"))for(const t of z.prev(e,ys))t.classList.add(_s)}_clearActiveClass(t){t.classList.remove(_s);const e=z.find(`${bs}.${_s}`,t);for(const t of e)t.classList.remove(_s)}static jQueryInterface(t){return this.each((function(){const e=Es.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}))}}N.on(window,gs,(()=>{for(const t of z.find('[data-bs-spy="scroll"]'))Es.getOrCreateInstance(t)})),m(Es);const Ts=".bs.tab",Cs=`hide${Ts}`,Os=`hidden${Ts}`,xs=`show${Ts}`,ks=`shown${Ts}`,Ls=`click${Ts}`,Ss=`keydown${Ts}`,Ds=`load${Ts}`,$s="ArrowLeft",Is="ArrowRight",Ns="ArrowUp",Ps="ArrowDown",Ms="Home",js="End",Fs="active",Hs="fade",Ws="show",Bs=":not(.dropdown-toggle)",zs='[data-bs-toggle="tab"], [data-bs-toggle="pill"], [data-bs-toggle="list"]',Rs=`.nav-link${Bs}, .list-group-item${Bs}, [role="tab"]${Bs}, ${zs}`,qs=`.${Fs}[data-bs-toggle="tab"], .${Fs}[data-bs-toggle="pill"], .${Fs}[data-bs-toggle="list"]`;class Vs extends W{constructor(t){super(t),this._parent=this._element.closest('.list-group, .nav, [role="tablist"]'),this._parent&&(this._setInitialAttributes(this._parent,this._getChildren()),N.on(this._element,Ss,(t=>this._keydown(t))))}static get NAME(){return"tab"}show(){const t=this._element;if(this._elemIsActive(t))return;const e=this._getActiveElem(),i=e?N.trigger(e,Cs,{relatedTarget:t}):null;N.trigger(t,xs,{relatedTarget:e}).defaultPrevented||i&&i.defaultPrevented||(this._deactivate(e,t),this._activate(t,e))}_activate(t,e){t&&(t.classList.add(Fs),this._activate(z.getElementFromSelector(t)),this._queueCallback((()=>{"tab"===t.getAttribute("role")?(t.removeAttribute("tabindex"),t.setAttribute("aria-selected",!0),this._toggleDropDown(t,!0),N.trigger(t,ks,{relatedTarget:e})):t.classList.add(Ws)}),t,t.classList.contains(Hs)))}_deactivate(t,e){t&&(t.classList.remove(Fs),t.blur(),this._deactivate(z.getElementFromSelector(t)),this._queueCallback((()=>{"tab"===t.getAttribute("role")?(t.setAttribute("aria-selected",!1),t.setAttribute("tabindex","-1"),this._toggleDropDown(t,!1),N.trigger(t,Os,{relatedTarget:e})):t.classList.remove(Ws)}),t,t.classList.contains(Hs)))}_keydown(t){if(![$s,Is,Ns,Ps,Ms,js].includes(t.key))return;t.stopPropagation(),t.preventDefault();const e=this._getChildren().filter((t=>!l(t)));let i;if([Ms,js].includes(t.key))i=e[t.key===Ms?0:e.length-1];else{const n=[Is,Ps].includes(t.key);i=b(e,t.target,n,!0)}i&&(i.focus({preventScroll:!0}),Vs.getOrCreateInstance(i).show())}_getChildren(){return z.find(Rs,this._parent)}_getActiveElem(){return this._getChildren().find((t=>this._elemIsActive(t)))||null}_setInitialAttributes(t,e){this._setAttributeIfNotExists(t,"role","tablist");for(const t of e)this._setInitialAttributesOnChild(t)}_setInitialAttributesOnChild(t){t=this._getInnerElement(t);const e=this._elemIsActive(t),i=this._getOuterElement(t);t.setAttribute("aria-selected",e),i!==t&&this._setAttributeIfNotExists(i,"role","presentation"),e||t.setAttribute("tabindex","-1"),this._setAttributeIfNotExists(t,"role","tab"),this._setInitialAttributesOnTargetPanel(t)}_setInitialAttributesOnTargetPanel(t){const e=z.getElementFromSelector(t);e&&(this._setAttributeIfNotExists(e,"role","tabpanel"),t.id&&this._setAttributeIfNotExists(e,"aria-labelledby",`${t.id}`))}_toggleDropDown(t,e){const i=this._getOuterElement(t);if(!i.classList.contains("dropdown"))return;const n=(t,n)=>{const s=z.findOne(t,i);s&&s.classList.toggle(n,e)};n(".dropdown-toggle",Fs),n(".dropdown-menu",Ws),i.setAttribute("aria-expanded",e)}_setAttributeIfNotExists(t,e,i){t.hasAttribute(e)||t.setAttribute(e,i)}_elemIsActive(t){return t.classList.contains(Fs)}_getInnerElement(t){return t.matches(Rs)?t:z.findOne(Rs,t)}_getOuterElement(t){return t.closest(".nav-item, .list-group-item")||t}static jQueryInterface(t){return this.each((function(){const e=Vs.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t]()}}))}}N.on(document,Ls,zs,(function(t){["A","AREA"].includes(this.tagName)&&t.preventDefault(),l(this)||Vs.getOrCreateInstance(this).show()})),N.on(window,Ds,(()=>{for(const t of z.find(qs))Vs.getOrCreateInstance(t)})),m(Vs);const Ks=".bs.toast",Qs=`mouseover${Ks}`,Xs=`mouseout${Ks}`,Ys=`focusin${Ks}`,Us=`focusout${Ks}`,Gs=`hide${Ks}`,Js=`hidden${Ks}`,Zs=`show${Ks}`,to=`shown${Ks}`,eo="hide",io="show",no="showing",so={animation:"boolean",autohide:"boolean",delay:"number"},oo={animation:!0,autohide:!0,delay:5e3};class ro extends W{constructor(t,e){super(t,e),this._timeout=null,this._hasMouseInteraction=!1,this._hasKeyboardInteraction=!1,this._setListeners()}static get Default(){return oo}static get DefaultType(){return so}static get NAME(){return"toast"}show(){N.trigger(this._element,Zs).defaultPrevented||(this._clearTimeout(),this._config.animation&&this._element.classList.add("fade"),this._element.classList.remove(eo),d(this._element),this._element.classList.add(io,no),this._queueCallback((()=>{this._element.classList.remove(no),N.trigger(this._element,to),this._maybeScheduleHide()}),this._element,this._config.animation))}hide(){this.isShown()&&(N.trigger(this._element,Gs).defaultPrevented||(this._element.classList.add(no),this._queueCallback((()=>{this._element.classList.add(eo),this._element.classList.remove(no,io),N.trigger(this._element,Js)}),this._element,this._config.animation)))}dispose(){this._clearTimeout(),this.isShown()&&this._element.classList.remove(io),super.dispose()}isShown(){return this._element.classList.contains(io)}_maybeScheduleHide(){this._config.autohide&&(this._hasMouseInteraction||this._hasKeyboardInteraction||(this._timeout=setTimeout((()=>{this.hide()}),this._config.delay)))}_onInteraction(t,e){switch(t.type){case"mouseover":case"mouseout":this._hasMouseInteraction=e;break;case"focusin":case"focusout":this._hasKeyboardInteraction=e}if(e)return void this._clearTimeout();const i=t.relatedTarget;this._element===i||this._element.contains(i)||this._maybeScheduleHide()}_setListeners(){N.on(this._element,Qs,(t=>this._onInteraction(t,!0))),N.on(this._element,Xs,(t=>this._onInteraction(t,!1))),N.on(this._element,Ys,(t=>this._onInteraction(t,!0))),N.on(this._element,Us,(t=>this._onInteraction(t,!1)))}_clearTimeout(){clearTimeout(this._timeout),this._timeout=null}static jQueryInterface(t){return this.each((function(){const e=ro.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}return R(ro),m(ro),{Alert:Q,Button:Y,Carousel:xt,Collapse:Bt,Dropdown:qi,Modal:On,Offcanvas:qn,Popover:us,ScrollSpy:Es,Tab:Vs,Toast:ro,Tooltip:cs}})); +//# sourceMappingURL=bootstrap.bundle.min.js.map \ No newline at end of file diff --git a/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js.map b/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js.map new file mode 100644 index 00000000..3863da8b --- /dev/null +++ b/dev/deps/bootstrap-5.3.1/bootstrap.bundle.min.js.map @@ -0,0 +1 @@ +{"version":3,"names":["elementMap","Map","Data","set","element","key","instance","has","instanceMap","get","size","console","error","Array","from","keys","remove","delete","TRANSITION_END","parseSelector","selector","window","CSS","escape","replace","match","id","triggerTransitionEnd","dispatchEvent","Event","isElement","object","jquery","nodeType","getElement","length","document","querySelector","isVisible","getClientRects","elementIsVisible","getComputedStyle","getPropertyValue","closedDetails","closest","summary","parentNode","isDisabled","Node","ELEMENT_NODE","classList","contains","disabled","hasAttribute","getAttribute","findShadowRoot","documentElement","attachShadow","getRootNode","root","ShadowRoot","noop","reflow","offsetHeight","getjQuery","jQuery","body","DOMContentLoadedCallbacks","isRTL","dir","defineJQueryPlugin","plugin","callback","$","name","NAME","JQUERY_NO_CONFLICT","fn","jQueryInterface","Constructor","noConflict","readyState","addEventListener","push","execute","possibleCallback","args","defaultValue","executeAfterTransition","transitionElement","waitForTransition","emulatedDuration","transitionDuration","transitionDelay","floatTransitionDuration","Number","parseFloat","floatTransitionDelay","split","getTransitionDurationFromElement","called","handler","target","removeEventListener","setTimeout","getNextActiveElement","list","activeElement","shouldGetNext","isCycleAllowed","listLength","index","indexOf","Math","max","min","namespaceRegex","stripNameRegex","stripUidRegex","eventRegistry","uidEvent","customEvents","mouseenter","mouseleave","nativeEvents","Set","makeEventUid","uid","getElementEvents","findHandler","events","callable","delegationSelector","Object","values","find","event","normalizeParameters","originalTypeEvent","delegationFunction","isDelegated","typeEvent","getTypeEvent","addHandler","oneOff","wrapFunction","relatedTarget","delegateTarget","call","this","handlers","previousFunction","domElements","querySelectorAll","domElement","hydrateObj","EventHandler","off","type","apply","bootstrapDelegationHandler","bootstrapHandler","removeHandler","Boolean","removeNamespacedHandlers","namespace","storeElementEvent","handlerKey","entries","includes","on","one","inNamespace","isNamespace","startsWith","elementEvent","slice","keyHandlers","trigger","jQueryEvent","bubbles","nativeDispatch","defaultPrevented","isPropagationStopped","isImmediatePropagationStopped","isDefaultPrevented","evt","cancelable","preventDefault","obj","meta","value","_unused","defineProperty","configurable","normalizeData","toString","JSON","parse","decodeURIComponent","normalizeDataKey","chr","toLowerCase","Manipulator","setDataAttribute","setAttribute","removeDataAttribute","removeAttribute","getDataAttributes","attributes","bsKeys","dataset","filter","pureKey","charAt","getDataAttribute","Config","Default","DefaultType","Error","_getConfig","config","_mergeConfigObj","_configAfterMerge","_typeCheckConfig","jsonConfig","constructor","configTypes","property","expectedTypes","valueType","prototype","RegExp","test","TypeError","toUpperCase","BaseComponent","super","_element","_config","DATA_KEY","dispose","EVENT_KEY","propertyName","getOwnPropertyNames","_queueCallback","isAnimated","getInstance","getOrCreateInstance","VERSION","eventName","getSelector","hrefAttribute","trim","SelectorEngine","concat","Element","findOne","children","child","matches","parents","ancestor","prev","previous","previousElementSibling","next","nextElementSibling","focusableChildren","focusables","map","join","el","getSelectorFromElement","getElementFromSelector","getMultipleElementsFromSelector","enableDismissTrigger","component","method","clickEvent","tagName","EVENT_CLOSE","EVENT_CLOSED","Alert","close","_destroyElement","each","data","undefined","SELECTOR_DATA_TOGGLE","Button","toggle","button","EVENT_TOUCHSTART","EVENT_TOUCHMOVE","EVENT_TOUCHEND","EVENT_POINTERDOWN","EVENT_POINTERUP","endCallback","leftCallback","rightCallback","Swipe","isSupported","_deltaX","_supportPointerEvents","PointerEvent","_initEvents","_start","_eventIsPointerPenTouch","clientX","touches","_end","_handleSwipe","_move","absDeltaX","abs","direction","add","pointerType","navigator","maxTouchPoints","DATA_API_KEY","ORDER_NEXT","ORDER_PREV","DIRECTION_LEFT","DIRECTION_RIGHT","EVENT_SLIDE","EVENT_SLID","EVENT_KEYDOWN","EVENT_MOUSEENTER","EVENT_MOUSELEAVE","EVENT_DRAG_START","EVENT_LOAD_DATA_API","EVENT_CLICK_DATA_API","CLASS_NAME_CAROUSEL","CLASS_NAME_ACTIVE","SELECTOR_ACTIVE","SELECTOR_ITEM","SELECTOR_ACTIVE_ITEM","KEY_TO_DIRECTION","ArrowLeft","ArrowRight","interval","keyboard","pause","ride","touch","wrap","Carousel","_interval","_activeElement","_isSliding","touchTimeout","_swipeHelper","_indicatorsElement","_addEventListeners","cycle","_slide","nextWhenVisible","hidden","_clearInterval","_updateInterval","setInterval","_maybeEnableCycle","to","items","_getItems","activeIndex","_getItemIndex","_getActive","order","defaultInterval","_keydown","_addTouchEventListeners","img","swipeConfig","_directionToOrder","endCallBack","clearTimeout","_setActiveIndicatorElement","activeIndicator","newActiveIndicator","elementInterval","parseInt","isNext","nextElement","nextElementIndex","triggerEvent","_orderToDirection","isCycling","directionalClassName","orderClassName","completeCallBack","_isAnimated","clearInterval","carousel","slideIndex","carousels","EVENT_SHOW","EVENT_SHOWN","EVENT_HIDE","EVENT_HIDDEN","CLASS_NAME_SHOW","CLASS_NAME_COLLAPSE","CLASS_NAME_COLLAPSING","CLASS_NAME_DEEPER_CHILDREN","parent","Collapse","_isTransitioning","_triggerArray","toggleList","elem","filterElement","foundElement","_initializeChildren","_addAriaAndCollapsedClass","_isShown","hide","show","activeChildren","_getFirstLevelChildren","activeInstance","dimension","_getDimension","style","scrollSize","complete","getBoundingClientRect","selected","triggerArray","isOpen","top","bottom","right","left","auto","basePlacements","start","end","clippingParents","viewport","popper","reference","variationPlacements","reduce","acc","placement","placements","beforeRead","read","afterRead","beforeMain","main","afterMain","beforeWrite","write","afterWrite","modifierPhases","getNodeName","nodeName","getWindow","node","ownerDocument","defaultView","isHTMLElement","HTMLElement","isShadowRoot","applyStyles$1","enabled","phase","_ref","state","elements","forEach","styles","assign","effect","_ref2","initialStyles","position","options","strategy","margin","arrow","hasOwnProperty","attribute","requires","getBasePlacement","round","getUAString","uaData","userAgentData","brands","isArray","item","brand","version","userAgent","isLayoutViewport","includeScale","isFixedStrategy","clientRect","scaleX","scaleY","offsetWidth","width","height","visualViewport","addVisualOffsets","x","offsetLeft","y","offsetTop","getLayoutRect","rootNode","isSameNode","host","isTableElement","getDocumentElement","getParentNode","assignedSlot","getTrueOffsetParent","offsetParent","getOffsetParent","isFirefox","currentNode","css","transform","perspective","contain","willChange","getContainingBlock","getMainAxisFromPlacement","within","mathMax","mathMin","mergePaddingObject","paddingObject","expandToHashMap","hashMap","arrow$1","_state$modifiersData$","arrowElement","popperOffsets","modifiersData","basePlacement","axis","len","padding","rects","toPaddingObject","arrowRect","minProp","maxProp","endDiff","startDiff","arrowOffsetParent","clientSize","clientHeight","clientWidth","centerToReference","center","offset","axisProp","centerOffset","_options$element","requiresIfExists","getVariation","unsetSides","mapToStyles","_Object$assign2","popperRect","variation","offsets","gpuAcceleration","adaptive","roundOffsets","isFixed","_offsets$x","_offsets$y","_ref3","hasX","hasY","sideX","sideY","win","heightProp","widthProp","_Object$assign","commonStyles","_ref4","dpr","devicePixelRatio","roundOffsetsByDPR","computeStyles$1","_ref5","_options$gpuAccelerat","_options$adaptive","_options$roundOffsets","passive","eventListeners","_options$scroll","scroll","_options$resize","resize","scrollParents","scrollParent","update","hash","getOppositePlacement","matched","getOppositeVariationPlacement","getWindowScroll","scrollLeft","pageXOffset","scrollTop","pageYOffset","getWindowScrollBarX","isScrollParent","_getComputedStyle","overflow","overflowX","overflowY","getScrollParent","listScrollParents","_element$ownerDocumen","isBody","updatedList","rectToClientRect","rect","getClientRectFromMixedType","clippingParent","html","layoutViewport","getViewportRect","clientTop","clientLeft","getInnerBoundingClientRect","winScroll","scrollWidth","scrollHeight","getDocumentRect","computeOffsets","commonX","commonY","mainAxis","detectOverflow","_options","_options$placement","_options$strategy","_options$boundary","boundary","_options$rootBoundary","rootBoundary","_options$elementConte","elementContext","_options$altBoundary","altBoundary","_options$padding","altContext","clippingClientRect","mainClippingParents","clipperElement","getClippingParents","firstClippingParent","clippingRect","accRect","getClippingRect","contextElement","referenceClientRect","popperClientRect","elementClientRect","overflowOffsets","offsetData","multiply","computeAutoPlacement","flipVariations","_options$allowedAutoP","allowedAutoPlacements","allPlacements","allowedPlacements","overflows","sort","a","b","flip$1","_skip","_options$mainAxis","checkMainAxis","_options$altAxis","altAxis","checkAltAxis","specifiedFallbackPlacements","fallbackPlacements","_options$flipVariatio","preferredPlacement","oppositePlacement","getExpandedFallbackPlacements","referenceRect","checksMap","makeFallbackChecks","firstFittingPlacement","i","_basePlacement","isStartVariation","isVertical","mainVariationSide","altVariationSide","checks","every","check","_loop","_i","fittingPlacement","reset","getSideOffsets","preventedOffsets","isAnySideFullyClipped","some","side","hide$1","preventOverflow","referenceOverflow","popperAltOverflow","referenceClippingOffsets","popperEscapeOffsets","isReferenceHidden","hasPopperEscaped","offset$1","_options$offset","invertDistance","skidding","distance","distanceAndSkiddingToXY","_data$state$placement","popperOffsets$1","preventOverflow$1","_options$tether","tether","_options$tetherOffset","tetherOffset","isBasePlacement","tetherOffsetValue","normalizedTetherOffsetValue","offsetModifierState","_offsetModifierState$","mainSide","altSide","additive","minLen","maxLen","arrowPaddingObject","arrowPaddingMin","arrowPaddingMax","arrowLen","minOffset","maxOffset","clientOffset","offsetModifierValue","tetherMax","preventedOffset","_offsetModifierState$2","_mainSide","_altSide","_offset","_len","_min","_max","isOriginSide","_offsetModifierValue","_tetherMin","_tetherMax","_preventedOffset","v","withinMaxClamp","getCompositeRect","elementOrVirtualElement","isOffsetParentAnElement","offsetParentIsScaled","isElementScaled","modifiers","visited","result","modifier","dep","depModifier","DEFAULT_OPTIONS","areValidElements","arguments","_key","popperGenerator","generatorOptions","_generatorOptions","_generatorOptions$def","defaultModifiers","_generatorOptions$def2","defaultOptions","pending","orderedModifiers","effectCleanupFns","isDestroyed","setOptions","setOptionsAction","cleanupModifierEffects","merged","orderModifiers","current","existing","m","_ref$options","cleanupFn","forceUpdate","_state$elements","_state$orderedModifie","_state$orderedModifie2","Promise","resolve","then","destroy","onFirstUpdate","createPopper","computeStyles","applyStyles","flip","ARROW_UP_KEY","ARROW_DOWN_KEY","EVENT_KEYDOWN_DATA_API","EVENT_KEYUP_DATA_API","SELECTOR_DATA_TOGGLE_SHOWN","SELECTOR_MENU","PLACEMENT_TOP","PLACEMENT_TOPEND","PLACEMENT_BOTTOM","PLACEMENT_BOTTOMEND","PLACEMENT_RIGHT","PLACEMENT_LEFT","autoClose","display","popperConfig","Dropdown","_popper","_parent","_menu","_inNavbar","_detectNavbar","_createPopper","focus","_completeHide","Popper","referenceElement","_getPopperConfig","_getPlacement","parentDropdown","isEnd","_getOffset","popperData","defaultBsPopperConfig","_selectMenuItem","clearMenus","openToggles","context","composedPath","isMenuTarget","dataApiKeydownHandler","isInput","isEscapeEvent","isUpOrDownEvent","getToggleButton","stopPropagation","EVENT_MOUSEDOWN","className","clickCallback","rootElement","Backdrop","_isAppended","_append","_getElement","_emulateAnimation","backdrop","createElement","append","EVENT_FOCUSIN","EVENT_KEYDOWN_TAB","TAB_NAV_BACKWARD","autofocus","trapElement","FocusTrap","_isActive","_lastTabNavDirection","activate","_handleFocusin","_handleKeydown","deactivate","shiftKey","SELECTOR_FIXED_CONTENT","SELECTOR_STICKY_CONTENT","PROPERTY_PADDING","PROPERTY_MARGIN","ScrollBarHelper","getWidth","documentWidth","innerWidth","_disableOverFlow","_setElementAttributes","calculatedValue","_resetElementAttributes","isOverflowing","_saveInitialAttribute","styleProperty","scrollbarWidth","_applyManipulationCallback","setProperty","actualValue","removeProperty","callBack","sel","EVENT_HIDE_PREVENTED","EVENT_RESIZE","EVENT_CLICK_DISMISS","EVENT_MOUSEDOWN_DISMISS","EVENT_KEYDOWN_DISMISS","CLASS_NAME_OPEN","CLASS_NAME_STATIC","Modal","_dialog","_backdrop","_initializeBackDrop","_focustrap","_initializeFocusTrap","_scrollBar","_adjustDialog","_showElement","_hideModal","handleUpdate","modalBody","transitionComplete","_triggerBackdropTransition","event2","_resetAdjustments","isModalOverflowing","initialOverflowY","isBodyOverflowing","paddingLeft","paddingRight","showEvent","alreadyOpen","CLASS_NAME_SHOWING","CLASS_NAME_HIDING","OPEN_SELECTOR","Offcanvas","blur","completeCallback","DefaultAllowlist","area","br","col","code","div","em","hr","h1","h2","h3","h4","h5","h6","li","ol","p","pre","s","small","span","sub","sup","strong","u","ul","uriAttributes","SAFE_URL_PATTERN","allowedAttribute","allowedAttributeList","attributeName","nodeValue","attributeRegex","regex","allowList","content","extraClass","sanitize","sanitizeFn","template","DefaultContentType","entry","TemplateFactory","getContent","_resolvePossibleFunction","hasContent","changeContent","_checkContent","toHtml","templateWrapper","innerHTML","_maybeSanitize","text","_setContent","arg","templateElement","_putElementInTemplate","textContent","unsafeHtml","sanitizeFunction","createdDocument","DOMParser","parseFromString","elementName","attributeList","allowedAttributes","sanitizeHtml","DISALLOWED_ATTRIBUTES","CLASS_NAME_FADE","SELECTOR_MODAL","EVENT_MODAL_HIDE","TRIGGER_HOVER","TRIGGER_FOCUS","AttachmentMap","AUTO","TOP","RIGHT","BOTTOM","LEFT","animation","container","customClass","delay","title","Tooltip","_isEnabled","_timeout","_isHovered","_activeTrigger","_templateFactory","_newContent","tip","_setListeners","_fixTitle","enable","disable","toggleEnabled","click","_leave","_enter","_hideModalHandler","_disposePopper","_isWithContent","isInTheDom","_getTipElement","_isWithActiveTrigger","_getTitle","_createTipElement","_getContentForTemplate","_getTemplateFactory","tipId","prefix","floor","random","getElementById","getUID","setContent","_initializeOnDelegatedTarget","_getDelegateConfig","attachment","triggers","eventIn","eventOut","_setTimeout","timeout","dataAttributes","dataAttribute","Popover","_getContent","EVENT_ACTIVATE","EVENT_CLICK","SELECTOR_TARGET_LINKS","SELECTOR_NAV_LINKS","SELECTOR_LINK_ITEMS","rootMargin","smoothScroll","threshold","ScrollSpy","_targetLinks","_observableSections","_rootElement","_activeTarget","_observer","_previousScrollData","visibleEntryTop","parentScrollTop","refresh","_initializeTargetsAndObservables","_maybeEnableSmoothScroll","disconnect","_getNewObserver","section","observe","observableSection","scrollTo","behavior","IntersectionObserver","_observerCallback","targetElement","_process","userScrollsDown","isIntersecting","_clearActiveClass","entryIsLowerThanPrevious","targetLinks","anchor","decodeURI","_activateParents","listGroup","activeNodes","spy","ARROW_LEFT_KEY","ARROW_RIGHT_KEY","HOME_KEY","END_KEY","NOT_SELECTOR_DROPDOWN_TOGGLE","SELECTOR_INNER_ELEM","SELECTOR_DATA_TOGGLE_ACTIVE","Tab","_setInitialAttributes","_getChildren","innerElem","_elemIsActive","active","_getActiveElem","hideEvent","_deactivate","_activate","relatedElem","_toggleDropDown","nextActiveElement","preventScroll","_setAttributeIfNotExists","_setInitialAttributesOnChild","_getInnerElement","isActive","outerElem","_getOuterElement","_setInitialAttributesOnTargetPanel","open","EVENT_MOUSEOVER","EVENT_MOUSEOUT","EVENT_FOCUSOUT","CLASS_NAME_HIDE","autohide","Toast","_hasMouseInteraction","_hasKeyboardInteraction","_clearTimeout","_maybeScheduleHide","isShown","_onInteraction","isInteracting"],"sources":["../../js/src/dom/data.js","../../js/src/util/index.js","../../js/src/dom/event-handler.js","../../js/src/dom/manipulator.js","../../js/src/util/config.js","../../js/src/base-component.js","../../js/src/dom/selector-engine.js","../../js/src/util/component-functions.js","../../js/src/alert.js","../../js/src/button.js","../../js/src/util/swipe.js","../../js/src/carousel.js","../../js/src/collapse.js","../../node_modules/@popperjs/core/lib/enums.js","../../node_modules/@popperjs/core/lib/dom-utils/getNodeName.js","../../node_modules/@popperjs/core/lib/dom-utils/getWindow.js","../../node_modules/@popperjs/core/lib/dom-utils/instanceOf.js","../../node_modules/@popperjs/core/lib/modifiers/applyStyles.js","../../node_modules/@popperjs/core/lib/utils/getBasePlacement.js","../../node_modules/@popperjs/core/lib/utils/math.js","../../node_modules/@popperjs/core/lib/utils/userAgent.js","../../node_modules/@popperjs/core/lib/dom-utils/isLayoutViewport.js","../../node_modules/@popperjs/core/lib/dom-utils/getBoundingClientRect.js","../../node_modules/@popperjs/core/lib/dom-utils/getLayoutRect.js","../../node_modules/@popperjs/core/lib/dom-utils/contains.js","../../node_modules/@popperjs/core/lib/dom-utils/getComputedStyle.js","../../node_modules/@popperjs/core/lib/dom-utils/isTableElement.js","../../node_modules/@popperjs/core/lib/dom-utils/getDocumentElement.js","../../node_modules/@popperjs/core/lib/dom-utils/getParentNode.js","../../node_modules/@popperjs/core/lib/dom-utils/getOffsetParent.js","../../node_modules/@popperjs/core/lib/utils/getMainAxisFromPlacement.js","../../node_modules/@popperjs/core/lib/utils/within.js","../../node_modules/@popperjs/core/lib/utils/mergePaddingObject.js","../../node_modules/@popperjs/core/lib/utils/getFreshSideObject.js","../../node_modules/@popperjs/core/lib/utils/expandToHashMap.js","../../node_modules/@popperjs/core/lib/modifiers/arrow.js","../../node_modules/@popperjs/core/lib/utils/getVariation.js","../../node_modules/@popperjs/core/lib/modifiers/computeStyles.js","../../node_modules/@popperjs/core/lib/modifiers/eventListeners.js","../../node_modules/@popperjs/core/lib/utils/getOppositePlacement.js","../../node_modules/@popperjs/core/lib/utils/getOppositeVariationPlacement.js","../../node_modules/@popperjs/core/lib/dom-utils/getWindowScroll.js","../../node_modules/@popperjs/core/lib/dom-utils/getWindowScrollBarX.js","../../node_modules/@popperjs/core/lib/dom-utils/isScrollParent.js","../../node_modules/@popperjs/core/lib/dom-utils/getScrollParent.js","../../node_modules/@popperjs/core/lib/dom-utils/listScrollParents.js","../../node_modules/@popperjs/core/lib/utils/rectToClientRect.js","../../node_modules/@popperjs/core/lib/dom-utils/getClippingRect.js","../../node_modules/@popperjs/core/lib/dom-utils/getViewportRect.js","../../node_modules/@popperjs/core/lib/dom-utils/getDocumentRect.js","../../node_modules/@popperjs/core/lib/utils/computeOffsets.js","../../node_modules/@popperjs/core/lib/utils/detectOverflow.js","../../node_modules/@popperjs/core/lib/utils/computeAutoPlacement.js","../../node_modules/@popperjs/core/lib/modifiers/flip.js","../../node_modules/@popperjs/core/lib/modifiers/hide.js","../../node_modules/@popperjs/core/lib/modifiers/offset.js","../../node_modules/@popperjs/core/lib/modifiers/popperOffsets.js","../../node_modules/@popperjs/core/lib/modifiers/preventOverflow.js","../../node_modules/@popperjs/core/lib/utils/getAltAxis.js","../../node_modules/@popperjs/core/lib/dom-utils/getCompositeRect.js","../../node_modules/@popperjs/core/lib/dom-utils/getNodeScroll.js","../../node_modules/@popperjs/core/lib/dom-utils/getHTMLElementScroll.js","../../node_modules/@popperjs/core/lib/utils/orderModifiers.js","../../node_modules/@popperjs/core/lib/createPopper.js","../../node_modules/@popperjs/core/lib/utils/debounce.js","../../node_modules/@popperjs/core/lib/utils/mergeByName.js","../../node_modules/@popperjs/core/lib/popper-lite.js","../../node_modules/@popperjs/core/lib/popper.js","../../js/src/dropdown.js","../../js/src/util/backdrop.js","../../js/src/util/focustrap.js","../../js/src/util/scrollbar.js","../../js/src/modal.js","../../js/src/offcanvas.js","../../js/src/util/sanitizer.js","../../js/src/util/template-factory.js","../../js/src/tooltip.js","../../js/src/popover.js","../../js/src/scrollspy.js","../../js/src/tab.js","../../js/src/toast.js","../../js/index.umd.js"],"sourcesContent":["/**\n * --------------------------------------------------------------------------\n * Bootstrap dom/data.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\n/**\n * Constants\n */\n\nconst elementMap = new Map()\n\nexport default {\n set(element, key, instance) {\n if (!elementMap.has(element)) {\n elementMap.set(element, new Map())\n }\n\n const instanceMap = elementMap.get(element)\n\n // make it clear we only want one instance per element\n // can be removed later when multiple key/instances are fine to be used\n if (!instanceMap.has(key) && instanceMap.size !== 0) {\n // eslint-disable-next-line no-console\n console.error(`Bootstrap doesn't allow more than one instance per element. Bound instance: ${Array.from(instanceMap.keys())[0]}.`)\n return\n }\n\n instanceMap.set(key, instance)\n },\n\n get(element, key) {\n if (elementMap.has(element)) {\n return elementMap.get(element).get(key) || null\n }\n\n return null\n },\n\n remove(element, key) {\n if (!elementMap.has(element)) {\n return\n }\n\n const instanceMap = elementMap.get(element)\n\n instanceMap.delete(key)\n\n // free up element references if there are no instances left for an element\n if (instanceMap.size === 0) {\n elementMap.delete(element)\n }\n }\n}\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/index.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nconst MAX_UID = 1_000_000\nconst MILLISECONDS_MULTIPLIER = 1000\nconst TRANSITION_END = 'transitionend'\n\n/**\n * Properly escape IDs selectors to handle weird IDs\n * @param {string} selector\n * @returns {string}\n */\nconst parseSelector = selector => {\n if (selector && window.CSS && window.CSS.escape) {\n // document.querySelector needs escaping to handle IDs (html5+) containing for instance /\n selector = selector.replace(/#([^\\s\"#']+)/g, (match, id) => `#${CSS.escape(id)}`)\n }\n\n return selector\n}\n\n// Shout-out Angus Croll (https://goo.gl/pxwQGp)\nconst toType = object => {\n if (object === null || object === undefined) {\n return `${object}`\n }\n\n return Object.prototype.toString.call(object).match(/\\s([a-z]+)/i)[1].toLowerCase()\n}\n\n/**\n * Public Util API\n */\n\nconst getUID = prefix => {\n do {\n prefix += Math.floor(Math.random() * MAX_UID)\n } while (document.getElementById(prefix))\n\n return prefix\n}\n\nconst getTransitionDurationFromElement = element => {\n if (!element) {\n return 0\n }\n\n // Get transition-duration of the element\n let { transitionDuration, transitionDelay } = window.getComputedStyle(element)\n\n const floatTransitionDuration = Number.parseFloat(transitionDuration)\n const floatTransitionDelay = Number.parseFloat(transitionDelay)\n\n // Return 0 if element or transition duration is not found\n if (!floatTransitionDuration && !floatTransitionDelay) {\n return 0\n }\n\n // If multiple durations are defined, take the first\n transitionDuration = transitionDuration.split(',')[0]\n transitionDelay = transitionDelay.split(',')[0]\n\n return (Number.parseFloat(transitionDuration) + Number.parseFloat(transitionDelay)) * MILLISECONDS_MULTIPLIER\n}\n\nconst triggerTransitionEnd = element => {\n element.dispatchEvent(new Event(TRANSITION_END))\n}\n\nconst isElement = object => {\n if (!object || typeof object !== 'object') {\n return false\n }\n\n if (typeof object.jquery !== 'undefined') {\n object = object[0]\n }\n\n return typeof object.nodeType !== 'undefined'\n}\n\nconst getElement = object => {\n // it's a jQuery object or a node element\n if (isElement(object)) {\n return object.jquery ? object[0] : object\n }\n\n if (typeof object === 'string' && object.length > 0) {\n return document.querySelector(parseSelector(object))\n }\n\n return null\n}\n\nconst isVisible = element => {\n if (!isElement(element) || element.getClientRects().length === 0) {\n return false\n }\n\n const elementIsVisible = getComputedStyle(element).getPropertyValue('visibility') === 'visible'\n // Handle `details` element as its content may falsie appear visible when it is closed\n const closedDetails = element.closest('details:not([open])')\n\n if (!closedDetails) {\n return elementIsVisible\n }\n\n if (closedDetails !== element) {\n const summary = element.closest('summary')\n if (summary && summary.parentNode !== closedDetails) {\n return false\n }\n\n if (summary === null) {\n return false\n }\n }\n\n return elementIsVisible\n}\n\nconst isDisabled = element => {\n if (!element || element.nodeType !== Node.ELEMENT_NODE) {\n return true\n }\n\n if (element.classList.contains('disabled')) {\n return true\n }\n\n if (typeof element.disabled !== 'undefined') {\n return element.disabled\n }\n\n return element.hasAttribute('disabled') && element.getAttribute('disabled') !== 'false'\n}\n\nconst findShadowRoot = element => {\n if (!document.documentElement.attachShadow) {\n return null\n }\n\n // Can find the shadow root otherwise it'll return the document\n if (typeof element.getRootNode === 'function') {\n const root = element.getRootNode()\n return root instanceof ShadowRoot ? root : null\n }\n\n if (element instanceof ShadowRoot) {\n return element\n }\n\n // when we don't find a shadow root\n if (!element.parentNode) {\n return null\n }\n\n return findShadowRoot(element.parentNode)\n}\n\nconst noop = () => {}\n\n/**\n * Trick to restart an element's animation\n *\n * @param {HTMLElement} element\n * @return void\n *\n * @see https://www.charistheo.io/blog/2021/02/restart-a-css-animation-with-javascript/#restarting-a-css-animation\n */\nconst reflow = element => {\n element.offsetHeight // eslint-disable-line no-unused-expressions\n}\n\nconst getjQuery = () => {\n if (window.jQuery && !document.body.hasAttribute('data-bs-no-jquery')) {\n return window.jQuery\n }\n\n return null\n}\n\nconst DOMContentLoadedCallbacks = []\n\nconst onDOMContentLoaded = callback => {\n if (document.readyState === 'loading') {\n // add listener on the first call when the document is in loading state\n if (!DOMContentLoadedCallbacks.length) {\n document.addEventListener('DOMContentLoaded', () => {\n for (const callback of DOMContentLoadedCallbacks) {\n callback()\n }\n })\n }\n\n DOMContentLoadedCallbacks.push(callback)\n } else {\n callback()\n }\n}\n\nconst isRTL = () => document.documentElement.dir === 'rtl'\n\nconst defineJQueryPlugin = plugin => {\n onDOMContentLoaded(() => {\n const $ = getjQuery()\n /* istanbul ignore if */\n if ($) {\n const name = plugin.NAME\n const JQUERY_NO_CONFLICT = $.fn[name]\n $.fn[name] = plugin.jQueryInterface\n $.fn[name].Constructor = plugin\n $.fn[name].noConflict = () => {\n $.fn[name] = JQUERY_NO_CONFLICT\n return plugin.jQueryInterface\n }\n }\n })\n}\n\nconst execute = (possibleCallback, args = [], defaultValue = possibleCallback) => {\n return typeof possibleCallback === 'function' ? possibleCallback(...args) : defaultValue\n}\n\nconst executeAfterTransition = (callback, transitionElement, waitForTransition = true) => {\n if (!waitForTransition) {\n execute(callback)\n return\n }\n\n const durationPadding = 5\n const emulatedDuration = getTransitionDurationFromElement(transitionElement) + durationPadding\n\n let called = false\n\n const handler = ({ target }) => {\n if (target !== transitionElement) {\n return\n }\n\n called = true\n transitionElement.removeEventListener(TRANSITION_END, handler)\n execute(callback)\n }\n\n transitionElement.addEventListener(TRANSITION_END, handler)\n setTimeout(() => {\n if (!called) {\n triggerTransitionEnd(transitionElement)\n }\n }, emulatedDuration)\n}\n\n/**\n * Return the previous/next element of a list.\n *\n * @param {array} list The list of elements\n * @param activeElement The active element\n * @param shouldGetNext Choose to get next or previous element\n * @param isCycleAllowed\n * @return {Element|elem} The proper element\n */\nconst getNextActiveElement = (list, activeElement, shouldGetNext, isCycleAllowed) => {\n const listLength = list.length\n let index = list.indexOf(activeElement)\n\n // if the element does not exist in the list return an element\n // depending on the direction and if cycle is allowed\n if (index === -1) {\n return !shouldGetNext && isCycleAllowed ? list[listLength - 1] : list[0]\n }\n\n index += shouldGetNext ? 1 : -1\n\n if (isCycleAllowed) {\n index = (index + listLength) % listLength\n }\n\n return list[Math.max(0, Math.min(index, listLength - 1))]\n}\n\nexport {\n defineJQueryPlugin,\n execute,\n executeAfterTransition,\n findShadowRoot,\n getElement,\n getjQuery,\n getNextActiveElement,\n getTransitionDurationFromElement,\n getUID,\n isDisabled,\n isElement,\n isRTL,\n isVisible,\n noop,\n onDOMContentLoaded,\n parseSelector,\n reflow,\n triggerTransitionEnd,\n toType\n}\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap dom/event-handler.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport { getjQuery } from '../util/index.js'\n\n/**\n * Constants\n */\n\nconst namespaceRegex = /[^.]*(?=\\..*)\\.|.*/\nconst stripNameRegex = /\\..*/\nconst stripUidRegex = /::\\d+$/\nconst eventRegistry = {} // Events storage\nlet uidEvent = 1\nconst customEvents = {\n mouseenter: 'mouseover',\n mouseleave: 'mouseout'\n}\n\nconst nativeEvents = new Set([\n 'click',\n 'dblclick',\n 'mouseup',\n 'mousedown',\n 'contextmenu',\n 'mousewheel',\n 'DOMMouseScroll',\n 'mouseover',\n 'mouseout',\n 'mousemove',\n 'selectstart',\n 'selectend',\n 'keydown',\n 'keypress',\n 'keyup',\n 'orientationchange',\n 'touchstart',\n 'touchmove',\n 'touchend',\n 'touchcancel',\n 'pointerdown',\n 'pointermove',\n 'pointerup',\n 'pointerleave',\n 'pointercancel',\n 'gesturestart',\n 'gesturechange',\n 'gestureend',\n 'focus',\n 'blur',\n 'change',\n 'reset',\n 'select',\n 'submit',\n 'focusin',\n 'focusout',\n 'load',\n 'unload',\n 'beforeunload',\n 'resize',\n 'move',\n 'DOMContentLoaded',\n 'readystatechange',\n 'error',\n 'abort',\n 'scroll'\n])\n\n/**\n * Private methods\n */\n\nfunction makeEventUid(element, uid) {\n return (uid && `${uid}::${uidEvent++}`) || element.uidEvent || uidEvent++\n}\n\nfunction getElementEvents(element) {\n const uid = makeEventUid(element)\n\n element.uidEvent = uid\n eventRegistry[uid] = eventRegistry[uid] || {}\n\n return eventRegistry[uid]\n}\n\nfunction bootstrapHandler(element, fn) {\n return function handler(event) {\n hydrateObj(event, { delegateTarget: element })\n\n if (handler.oneOff) {\n EventHandler.off(element, event.type, fn)\n }\n\n return fn.apply(element, [event])\n }\n}\n\nfunction bootstrapDelegationHandler(element, selector, fn) {\n return function handler(event) {\n const domElements = element.querySelectorAll(selector)\n\n for (let { target } = event; target && target !== this; target = target.parentNode) {\n for (const domElement of domElements) {\n if (domElement !== target) {\n continue\n }\n\n hydrateObj(event, { delegateTarget: target })\n\n if (handler.oneOff) {\n EventHandler.off(element, event.type, selector, fn)\n }\n\n return fn.apply(target, [event])\n }\n }\n }\n}\n\nfunction findHandler(events, callable, delegationSelector = null) {\n return Object.values(events)\n .find(event => event.callable === callable && event.delegationSelector === delegationSelector)\n}\n\nfunction normalizeParameters(originalTypeEvent, handler, delegationFunction) {\n const isDelegated = typeof handler === 'string'\n // TODO: tooltip passes `false` instead of selector, so we need to check\n const callable = isDelegated ? delegationFunction : (handler || delegationFunction)\n let typeEvent = getTypeEvent(originalTypeEvent)\n\n if (!nativeEvents.has(typeEvent)) {\n typeEvent = originalTypeEvent\n }\n\n return [isDelegated, callable, typeEvent]\n}\n\nfunction addHandler(element, originalTypeEvent, handler, delegationFunction, oneOff) {\n if (typeof originalTypeEvent !== 'string' || !element) {\n return\n }\n\n let [isDelegated, callable, typeEvent] = normalizeParameters(originalTypeEvent, handler, delegationFunction)\n\n // in case of mouseenter or mouseleave wrap the handler within a function that checks for its DOM position\n // this prevents the handler from being dispatched the same way as mouseover or mouseout does\n if (originalTypeEvent in customEvents) {\n const wrapFunction = fn => {\n return function (event) {\n if (!event.relatedTarget || (event.relatedTarget !== event.delegateTarget && !event.delegateTarget.contains(event.relatedTarget))) {\n return fn.call(this, event)\n }\n }\n }\n\n callable = wrapFunction(callable)\n }\n\n const events = getElementEvents(element)\n const handlers = events[typeEvent] || (events[typeEvent] = {})\n const previousFunction = findHandler(handlers, callable, isDelegated ? handler : null)\n\n if (previousFunction) {\n previousFunction.oneOff = previousFunction.oneOff && oneOff\n\n return\n }\n\n const uid = makeEventUid(callable, originalTypeEvent.replace(namespaceRegex, ''))\n const fn = isDelegated ?\n bootstrapDelegationHandler(element, handler, callable) :\n bootstrapHandler(element, callable)\n\n fn.delegationSelector = isDelegated ? handler : null\n fn.callable = callable\n fn.oneOff = oneOff\n fn.uidEvent = uid\n handlers[uid] = fn\n\n element.addEventListener(typeEvent, fn, isDelegated)\n}\n\nfunction removeHandler(element, events, typeEvent, handler, delegationSelector) {\n const fn = findHandler(events[typeEvent], handler, delegationSelector)\n\n if (!fn) {\n return\n }\n\n element.removeEventListener(typeEvent, fn, Boolean(delegationSelector))\n delete events[typeEvent][fn.uidEvent]\n}\n\nfunction removeNamespacedHandlers(element, events, typeEvent, namespace) {\n const storeElementEvent = events[typeEvent] || {}\n\n for (const [handlerKey, event] of Object.entries(storeElementEvent)) {\n if (handlerKey.includes(namespace)) {\n removeHandler(element, events, typeEvent, event.callable, event.delegationSelector)\n }\n }\n}\n\nfunction getTypeEvent(event) {\n // allow to get the native events from namespaced events ('click.bs.button' --> 'click')\n event = event.replace(stripNameRegex, '')\n return customEvents[event] || event\n}\n\nconst EventHandler = {\n on(element, event, handler, delegationFunction) {\n addHandler(element, event, handler, delegationFunction, false)\n },\n\n one(element, event, handler, delegationFunction) {\n addHandler(element, event, handler, delegationFunction, true)\n },\n\n off(element, originalTypeEvent, handler, delegationFunction) {\n if (typeof originalTypeEvent !== 'string' || !element) {\n return\n }\n\n const [isDelegated, callable, typeEvent] = normalizeParameters(originalTypeEvent, handler, delegationFunction)\n const inNamespace = typeEvent !== originalTypeEvent\n const events = getElementEvents(element)\n const storeElementEvent = events[typeEvent] || {}\n const isNamespace = originalTypeEvent.startsWith('.')\n\n if (typeof callable !== 'undefined') {\n // Simplest case: handler is passed, remove that listener ONLY.\n if (!Object.keys(storeElementEvent).length) {\n return\n }\n\n removeHandler(element, events, typeEvent, callable, isDelegated ? handler : null)\n return\n }\n\n if (isNamespace) {\n for (const elementEvent of Object.keys(events)) {\n removeNamespacedHandlers(element, events, elementEvent, originalTypeEvent.slice(1))\n }\n }\n\n for (const [keyHandlers, event] of Object.entries(storeElementEvent)) {\n const handlerKey = keyHandlers.replace(stripUidRegex, '')\n\n if (!inNamespace || originalTypeEvent.includes(handlerKey)) {\n removeHandler(element, events, typeEvent, event.callable, event.delegationSelector)\n }\n }\n },\n\n trigger(element, event, args) {\n if (typeof event !== 'string' || !element) {\n return null\n }\n\n const $ = getjQuery()\n const typeEvent = getTypeEvent(event)\n const inNamespace = event !== typeEvent\n\n let jQueryEvent = null\n let bubbles = true\n let nativeDispatch = true\n let defaultPrevented = false\n\n if (inNamespace && $) {\n jQueryEvent = $.Event(event, args)\n\n $(element).trigger(jQueryEvent)\n bubbles = !jQueryEvent.isPropagationStopped()\n nativeDispatch = !jQueryEvent.isImmediatePropagationStopped()\n defaultPrevented = jQueryEvent.isDefaultPrevented()\n }\n\n const evt = hydrateObj(new Event(event, { bubbles, cancelable: true }), args)\n\n if (defaultPrevented) {\n evt.preventDefault()\n }\n\n if (nativeDispatch) {\n element.dispatchEvent(evt)\n }\n\n if (evt.defaultPrevented && jQueryEvent) {\n jQueryEvent.preventDefault()\n }\n\n return evt\n }\n}\n\nfunction hydrateObj(obj, meta = {}) {\n for (const [key, value] of Object.entries(meta)) {\n try {\n obj[key] = value\n } catch {\n Object.defineProperty(obj, key, {\n configurable: true,\n get() {\n return value\n }\n })\n }\n }\n\n return obj\n}\n\nexport default EventHandler\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap dom/manipulator.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nfunction normalizeData(value) {\n if (value === 'true') {\n return true\n }\n\n if (value === 'false') {\n return false\n }\n\n if (value === Number(value).toString()) {\n return Number(value)\n }\n\n if (value === '' || value === 'null') {\n return null\n }\n\n if (typeof value !== 'string') {\n return value\n }\n\n try {\n return JSON.parse(decodeURIComponent(value))\n } catch {\n return value\n }\n}\n\nfunction normalizeDataKey(key) {\n return key.replace(/[A-Z]/g, chr => `-${chr.toLowerCase()}`)\n}\n\nconst Manipulator = {\n setDataAttribute(element, key, value) {\n element.setAttribute(`data-bs-${normalizeDataKey(key)}`, value)\n },\n\n removeDataAttribute(element, key) {\n element.removeAttribute(`data-bs-${normalizeDataKey(key)}`)\n },\n\n getDataAttributes(element) {\n if (!element) {\n return {}\n }\n\n const attributes = {}\n const bsKeys = Object.keys(element.dataset).filter(key => key.startsWith('bs') && !key.startsWith('bsConfig'))\n\n for (const key of bsKeys) {\n let pureKey = key.replace(/^bs/, '')\n pureKey = pureKey.charAt(0).toLowerCase() + pureKey.slice(1, pureKey.length)\n attributes[pureKey] = normalizeData(element.dataset[key])\n }\n\n return attributes\n },\n\n getDataAttribute(element, key) {\n return normalizeData(element.getAttribute(`data-bs-${normalizeDataKey(key)}`))\n }\n}\n\nexport default Manipulator\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/config.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport Manipulator from '../dom/manipulator.js'\nimport { isElement, toType } from './index.js'\n\n/**\n * Class definition\n */\n\nclass Config {\n // Getters\n static get Default() {\n return {}\n }\n\n static get DefaultType() {\n return {}\n }\n\n static get NAME() {\n throw new Error('You have to implement the static method \"NAME\", for each component!')\n }\n\n _getConfig(config) {\n config = this._mergeConfigObj(config)\n config = this._configAfterMerge(config)\n this._typeCheckConfig(config)\n return config\n }\n\n _configAfterMerge(config) {\n return config\n }\n\n _mergeConfigObj(config, element) {\n const jsonConfig = isElement(element) ? Manipulator.getDataAttribute(element, 'config') : {} // try to parse\n\n return {\n ...this.constructor.Default,\n ...(typeof jsonConfig === 'object' ? jsonConfig : {}),\n ...(isElement(element) ? Manipulator.getDataAttributes(element) : {}),\n ...(typeof config === 'object' ? config : {})\n }\n }\n\n _typeCheckConfig(config, configTypes = this.constructor.DefaultType) {\n for (const [property, expectedTypes] of Object.entries(configTypes)) {\n const value = config[property]\n const valueType = isElement(value) ? 'element' : toType(value)\n\n if (!new RegExp(expectedTypes).test(valueType)) {\n throw new TypeError(\n `${this.constructor.NAME.toUpperCase()}: Option \"${property}\" provided type \"${valueType}\" but expected type \"${expectedTypes}\".`\n )\n }\n }\n }\n}\n\nexport default Config\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap base-component.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport Data from './dom/data.js'\nimport EventHandler from './dom/event-handler.js'\nimport Config from './util/config.js'\nimport { executeAfterTransition, getElement } from './util/index.js'\n\n/**\n * Constants\n */\n\nconst VERSION = '5.3.1'\n\n/**\n * Class definition\n */\n\nclass BaseComponent extends Config {\n constructor(element, config) {\n super()\n\n element = getElement(element)\n if (!element) {\n return\n }\n\n this._element = element\n this._config = this._getConfig(config)\n\n Data.set(this._element, this.constructor.DATA_KEY, this)\n }\n\n // Public\n dispose() {\n Data.remove(this._element, this.constructor.DATA_KEY)\n EventHandler.off(this._element, this.constructor.EVENT_KEY)\n\n for (const propertyName of Object.getOwnPropertyNames(this)) {\n this[propertyName] = null\n }\n }\n\n _queueCallback(callback, element, isAnimated = true) {\n executeAfterTransition(callback, element, isAnimated)\n }\n\n _getConfig(config) {\n config = this._mergeConfigObj(config, this._element)\n config = this._configAfterMerge(config)\n this._typeCheckConfig(config)\n return config\n }\n\n // Static\n static getInstance(element) {\n return Data.get(getElement(element), this.DATA_KEY)\n }\n\n static getOrCreateInstance(element, config = {}) {\n return this.getInstance(element) || new this(element, typeof config === 'object' ? config : null)\n }\n\n static get VERSION() {\n return VERSION\n }\n\n static get DATA_KEY() {\n return `bs.${this.NAME}`\n }\n\n static get EVENT_KEY() {\n return `.${this.DATA_KEY}`\n }\n\n static eventName(name) {\n return `${name}${this.EVENT_KEY}`\n }\n}\n\nexport default BaseComponent\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap dom/selector-engine.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport { isDisabled, isVisible, parseSelector } from '../util/index.js'\n\nconst getSelector = element => {\n let selector = element.getAttribute('data-bs-target')\n\n if (!selector || selector === '#') {\n let hrefAttribute = element.getAttribute('href')\n\n // The only valid content that could double as a selector are IDs or classes,\n // so everything starting with `#` or `.`. If a \"real\" URL is used as the selector,\n // `document.querySelector` will rightfully complain it is invalid.\n // See https://github.com/twbs/bootstrap/issues/32273\n if (!hrefAttribute || (!hrefAttribute.includes('#') && !hrefAttribute.startsWith('.'))) {\n return null\n }\n\n // Just in case some CMS puts out a full URL with the anchor appended\n if (hrefAttribute.includes('#') && !hrefAttribute.startsWith('#')) {\n hrefAttribute = `#${hrefAttribute.split('#')[1]}`\n }\n\n selector = hrefAttribute && hrefAttribute !== '#' ? hrefAttribute.trim() : null\n }\n\n return parseSelector(selector)\n}\n\nconst SelectorEngine = {\n find(selector, element = document.documentElement) {\n return [].concat(...Element.prototype.querySelectorAll.call(element, selector))\n },\n\n findOne(selector, element = document.documentElement) {\n return Element.prototype.querySelector.call(element, selector)\n },\n\n children(element, selector) {\n return [].concat(...element.children).filter(child => child.matches(selector))\n },\n\n parents(element, selector) {\n const parents = []\n let ancestor = element.parentNode.closest(selector)\n\n while (ancestor) {\n parents.push(ancestor)\n ancestor = ancestor.parentNode.closest(selector)\n }\n\n return parents\n },\n\n prev(element, selector) {\n let previous = element.previousElementSibling\n\n while (previous) {\n if (previous.matches(selector)) {\n return [previous]\n }\n\n previous = previous.previousElementSibling\n }\n\n return []\n },\n // TODO: this is now unused; remove later along with prev()\n next(element, selector) {\n let next = element.nextElementSibling\n\n while (next) {\n if (next.matches(selector)) {\n return [next]\n }\n\n next = next.nextElementSibling\n }\n\n return []\n },\n\n focusableChildren(element) {\n const focusables = [\n 'a',\n 'button',\n 'input',\n 'textarea',\n 'select',\n 'details',\n '[tabindex]',\n '[contenteditable=\"true\"]'\n ].map(selector => `${selector}:not([tabindex^=\"-\"])`).join(',')\n\n return this.find(focusables, element).filter(el => !isDisabled(el) && isVisible(el))\n },\n\n getSelectorFromElement(element) {\n const selector = getSelector(element)\n\n if (selector) {\n return SelectorEngine.findOne(selector) ? selector : null\n }\n\n return null\n },\n\n getElementFromSelector(element) {\n const selector = getSelector(element)\n\n return selector ? SelectorEngine.findOne(selector) : null\n },\n\n getMultipleElementsFromSelector(element) {\n const selector = getSelector(element)\n\n return selector ? SelectorEngine.find(selector) : []\n }\n}\n\nexport default SelectorEngine\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/component-functions.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport EventHandler from '../dom/event-handler.js'\nimport SelectorEngine from '../dom/selector-engine.js'\nimport { isDisabled } from './index.js'\n\nconst enableDismissTrigger = (component, method = 'hide') => {\n const clickEvent = `click.dismiss${component.EVENT_KEY}`\n const name = component.NAME\n\n EventHandler.on(document, clickEvent, `[data-bs-dismiss=\"${name}\"]`, function (event) {\n if (['A', 'AREA'].includes(this.tagName)) {\n event.preventDefault()\n }\n\n if (isDisabled(this)) {\n return\n }\n\n const target = SelectorEngine.getElementFromSelector(this) || this.closest(`.${name}`)\n const instance = component.getOrCreateInstance(target)\n\n // Method argument is left, for Alert and only, as it doesn't implement the 'hide' method\n instance[method]()\n })\n}\n\nexport {\n enableDismissTrigger\n}\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap alert.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport { enableDismissTrigger } from './util/component-functions.js'\nimport { defineJQueryPlugin } from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'alert'\nconst DATA_KEY = 'bs.alert'\nconst EVENT_KEY = `.${DATA_KEY}`\n\nconst EVENT_CLOSE = `close${EVENT_KEY}`\nconst EVENT_CLOSED = `closed${EVENT_KEY}`\nconst CLASS_NAME_FADE = 'fade'\nconst CLASS_NAME_SHOW = 'show'\n\n/**\n * Class definition\n */\n\nclass Alert extends BaseComponent {\n // Getters\n static get NAME() {\n return NAME\n }\n\n // Public\n close() {\n const closeEvent = EventHandler.trigger(this._element, EVENT_CLOSE)\n\n if (closeEvent.defaultPrevented) {\n return\n }\n\n this._element.classList.remove(CLASS_NAME_SHOW)\n\n const isAnimated = this._element.classList.contains(CLASS_NAME_FADE)\n this._queueCallback(() => this._destroyElement(), this._element, isAnimated)\n }\n\n // Private\n _destroyElement() {\n this._element.remove()\n EventHandler.trigger(this._element, EVENT_CLOSED)\n this.dispose()\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Alert.getOrCreateInstance(this)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (data[config] === undefined || config.startsWith('_') || config === 'constructor') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config](this)\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nenableDismissTrigger(Alert, 'close')\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Alert)\n\nexport default Alert\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap button.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport { defineJQueryPlugin } from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'button'\nconst DATA_KEY = 'bs.button'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\n\nconst CLASS_NAME_ACTIVE = 'active'\nconst SELECTOR_DATA_TOGGLE = '[data-bs-toggle=\"button\"]'\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\n\n/**\n * Class definition\n */\n\nclass Button extends BaseComponent {\n // Getters\n static get NAME() {\n return NAME\n }\n\n // Public\n toggle() {\n // Toggle class and sync the `aria-pressed` attribute with the return value of the `.toggle()` method\n this._element.setAttribute('aria-pressed', this._element.classList.toggle(CLASS_NAME_ACTIVE))\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Button.getOrCreateInstance(this)\n\n if (config === 'toggle') {\n data[config]()\n }\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_TOGGLE, event => {\n event.preventDefault()\n\n const button = event.target.closest(SELECTOR_DATA_TOGGLE)\n const data = Button.getOrCreateInstance(button)\n\n data.toggle()\n})\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Button)\n\nexport default Button\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/swipe.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport EventHandler from '../dom/event-handler.js'\nimport Config from './config.js'\nimport { execute } from './index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'swipe'\nconst EVENT_KEY = '.bs.swipe'\nconst EVENT_TOUCHSTART = `touchstart${EVENT_KEY}`\nconst EVENT_TOUCHMOVE = `touchmove${EVENT_KEY}`\nconst EVENT_TOUCHEND = `touchend${EVENT_KEY}`\nconst EVENT_POINTERDOWN = `pointerdown${EVENT_KEY}`\nconst EVENT_POINTERUP = `pointerup${EVENT_KEY}`\nconst POINTER_TYPE_TOUCH = 'touch'\nconst POINTER_TYPE_PEN = 'pen'\nconst CLASS_NAME_POINTER_EVENT = 'pointer-event'\nconst SWIPE_THRESHOLD = 40\n\nconst Default = {\n endCallback: null,\n leftCallback: null,\n rightCallback: null\n}\n\nconst DefaultType = {\n endCallback: '(function|null)',\n leftCallback: '(function|null)',\n rightCallback: '(function|null)'\n}\n\n/**\n * Class definition\n */\n\nclass Swipe extends Config {\n constructor(element, config) {\n super()\n this._element = element\n\n if (!element || !Swipe.isSupported()) {\n return\n }\n\n this._config = this._getConfig(config)\n this._deltaX = 0\n this._supportPointerEvents = Boolean(window.PointerEvent)\n this._initEvents()\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n dispose() {\n EventHandler.off(this._element, EVENT_KEY)\n }\n\n // Private\n _start(event) {\n if (!this._supportPointerEvents) {\n this._deltaX = event.touches[0].clientX\n\n return\n }\n\n if (this._eventIsPointerPenTouch(event)) {\n this._deltaX = event.clientX\n }\n }\n\n _end(event) {\n if (this._eventIsPointerPenTouch(event)) {\n this._deltaX = event.clientX - this._deltaX\n }\n\n this._handleSwipe()\n execute(this._config.endCallback)\n }\n\n _move(event) {\n this._deltaX = event.touches && event.touches.length > 1 ?\n 0 :\n event.touches[0].clientX - this._deltaX\n }\n\n _handleSwipe() {\n const absDeltaX = Math.abs(this._deltaX)\n\n if (absDeltaX <= SWIPE_THRESHOLD) {\n return\n }\n\n const direction = absDeltaX / this._deltaX\n\n this._deltaX = 0\n\n if (!direction) {\n return\n }\n\n execute(direction > 0 ? this._config.rightCallback : this._config.leftCallback)\n }\n\n _initEvents() {\n if (this._supportPointerEvents) {\n EventHandler.on(this._element, EVENT_POINTERDOWN, event => this._start(event))\n EventHandler.on(this._element, EVENT_POINTERUP, event => this._end(event))\n\n this._element.classList.add(CLASS_NAME_POINTER_EVENT)\n } else {\n EventHandler.on(this._element, EVENT_TOUCHSTART, event => this._start(event))\n EventHandler.on(this._element, EVENT_TOUCHMOVE, event => this._move(event))\n EventHandler.on(this._element, EVENT_TOUCHEND, event => this._end(event))\n }\n }\n\n _eventIsPointerPenTouch(event) {\n return this._supportPointerEvents && (event.pointerType === POINTER_TYPE_PEN || event.pointerType === POINTER_TYPE_TOUCH)\n }\n\n // Static\n static isSupported() {\n return 'ontouchstart' in document.documentElement || navigator.maxTouchPoints > 0\n }\n}\n\nexport default Swipe\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap carousel.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport Manipulator from './dom/manipulator.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport {\n defineJQueryPlugin,\n getNextActiveElement,\n isRTL,\n isVisible,\n reflow,\n triggerTransitionEnd\n} from './util/index.js'\nimport Swipe from './util/swipe.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'carousel'\nconst DATA_KEY = 'bs.carousel'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\n\nconst ARROW_LEFT_KEY = 'ArrowLeft'\nconst ARROW_RIGHT_KEY = 'ArrowRight'\nconst TOUCHEVENT_COMPAT_WAIT = 500 // Time for mouse compat events to fire after touch\n\nconst ORDER_NEXT = 'next'\nconst ORDER_PREV = 'prev'\nconst DIRECTION_LEFT = 'left'\nconst DIRECTION_RIGHT = 'right'\n\nconst EVENT_SLIDE = `slide${EVENT_KEY}`\nconst EVENT_SLID = `slid${EVENT_KEY}`\nconst EVENT_KEYDOWN = `keydown${EVENT_KEY}`\nconst EVENT_MOUSEENTER = `mouseenter${EVENT_KEY}`\nconst EVENT_MOUSELEAVE = `mouseleave${EVENT_KEY}`\nconst EVENT_DRAG_START = `dragstart${EVENT_KEY}`\nconst EVENT_LOAD_DATA_API = `load${EVENT_KEY}${DATA_API_KEY}`\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\n\nconst CLASS_NAME_CAROUSEL = 'carousel'\nconst CLASS_NAME_ACTIVE = 'active'\nconst CLASS_NAME_SLIDE = 'slide'\nconst CLASS_NAME_END = 'carousel-item-end'\nconst CLASS_NAME_START = 'carousel-item-start'\nconst CLASS_NAME_NEXT = 'carousel-item-next'\nconst CLASS_NAME_PREV = 'carousel-item-prev'\n\nconst SELECTOR_ACTIVE = '.active'\nconst SELECTOR_ITEM = '.carousel-item'\nconst SELECTOR_ACTIVE_ITEM = SELECTOR_ACTIVE + SELECTOR_ITEM\nconst SELECTOR_ITEM_IMG = '.carousel-item img'\nconst SELECTOR_INDICATORS = '.carousel-indicators'\nconst SELECTOR_DATA_SLIDE = '[data-bs-slide], [data-bs-slide-to]'\nconst SELECTOR_DATA_RIDE = '[data-bs-ride=\"carousel\"]'\n\nconst KEY_TO_DIRECTION = {\n [ARROW_LEFT_KEY]: DIRECTION_RIGHT,\n [ARROW_RIGHT_KEY]: DIRECTION_LEFT\n}\n\nconst Default = {\n interval: 5000,\n keyboard: true,\n pause: 'hover',\n ride: false,\n touch: true,\n wrap: true\n}\n\nconst DefaultType = {\n interval: '(number|boolean)', // TODO:v6 remove boolean support\n keyboard: 'boolean',\n pause: '(string|boolean)',\n ride: '(boolean|string)',\n touch: 'boolean',\n wrap: 'boolean'\n}\n\n/**\n * Class definition\n */\n\nclass Carousel extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n this._interval = null\n this._activeElement = null\n this._isSliding = false\n this.touchTimeout = null\n this._swipeHelper = null\n\n this._indicatorsElement = SelectorEngine.findOne(SELECTOR_INDICATORS, this._element)\n this._addEventListeners()\n\n if (this._config.ride === CLASS_NAME_CAROUSEL) {\n this.cycle()\n }\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n next() {\n this._slide(ORDER_NEXT)\n }\n\n nextWhenVisible() {\n // FIXME TODO use `document.visibilityState`\n // Don't call next when the page isn't visible\n // or the carousel or its parent isn't visible\n if (!document.hidden && isVisible(this._element)) {\n this.next()\n }\n }\n\n prev() {\n this._slide(ORDER_PREV)\n }\n\n pause() {\n if (this._isSliding) {\n triggerTransitionEnd(this._element)\n }\n\n this._clearInterval()\n }\n\n cycle() {\n this._clearInterval()\n this._updateInterval()\n\n this._interval = setInterval(() => this.nextWhenVisible(), this._config.interval)\n }\n\n _maybeEnableCycle() {\n if (!this._config.ride) {\n return\n }\n\n if (this._isSliding) {\n EventHandler.one(this._element, EVENT_SLID, () => this.cycle())\n return\n }\n\n this.cycle()\n }\n\n to(index) {\n const items = this._getItems()\n if (index > items.length - 1 || index < 0) {\n return\n }\n\n if (this._isSliding) {\n EventHandler.one(this._element, EVENT_SLID, () => this.to(index))\n return\n }\n\n const activeIndex = this._getItemIndex(this._getActive())\n if (activeIndex === index) {\n return\n }\n\n const order = index > activeIndex ? ORDER_NEXT : ORDER_PREV\n\n this._slide(order, items[index])\n }\n\n dispose() {\n if (this._swipeHelper) {\n this._swipeHelper.dispose()\n }\n\n super.dispose()\n }\n\n // Private\n _configAfterMerge(config) {\n config.defaultInterval = config.interval\n return config\n }\n\n _addEventListeners() {\n if (this._config.keyboard) {\n EventHandler.on(this._element, EVENT_KEYDOWN, event => this._keydown(event))\n }\n\n if (this._config.pause === 'hover') {\n EventHandler.on(this._element, EVENT_MOUSEENTER, () => this.pause())\n EventHandler.on(this._element, EVENT_MOUSELEAVE, () => this._maybeEnableCycle())\n }\n\n if (this._config.touch && Swipe.isSupported()) {\n this._addTouchEventListeners()\n }\n }\n\n _addTouchEventListeners() {\n for (const img of SelectorEngine.find(SELECTOR_ITEM_IMG, this._element)) {\n EventHandler.on(img, EVENT_DRAG_START, event => event.preventDefault())\n }\n\n const endCallBack = () => {\n if (this._config.pause !== 'hover') {\n return\n }\n\n // If it's a touch-enabled device, mouseenter/leave are fired as\n // part of the mouse compatibility events on first tap - the carousel\n // would stop cycling until user tapped out of it;\n // here, we listen for touchend, explicitly pause the carousel\n // (as if it's the second time we tap on it, mouseenter compat event\n // is NOT fired) and after a timeout (to allow for mouse compatibility\n // events to fire) we explicitly restart cycling\n\n this.pause()\n if (this.touchTimeout) {\n clearTimeout(this.touchTimeout)\n }\n\n this.touchTimeout = setTimeout(() => this._maybeEnableCycle(), TOUCHEVENT_COMPAT_WAIT + this._config.interval)\n }\n\n const swipeConfig = {\n leftCallback: () => this._slide(this._directionToOrder(DIRECTION_LEFT)),\n rightCallback: () => this._slide(this._directionToOrder(DIRECTION_RIGHT)),\n endCallback: endCallBack\n }\n\n this._swipeHelper = new Swipe(this._element, swipeConfig)\n }\n\n _keydown(event) {\n if (/input|textarea/i.test(event.target.tagName)) {\n return\n }\n\n const direction = KEY_TO_DIRECTION[event.key]\n if (direction) {\n event.preventDefault()\n this._slide(this._directionToOrder(direction))\n }\n }\n\n _getItemIndex(element) {\n return this._getItems().indexOf(element)\n }\n\n _setActiveIndicatorElement(index) {\n if (!this._indicatorsElement) {\n return\n }\n\n const activeIndicator = SelectorEngine.findOne(SELECTOR_ACTIVE, this._indicatorsElement)\n\n activeIndicator.classList.remove(CLASS_NAME_ACTIVE)\n activeIndicator.removeAttribute('aria-current')\n\n const newActiveIndicator = SelectorEngine.findOne(`[data-bs-slide-to=\"${index}\"]`, this._indicatorsElement)\n\n if (newActiveIndicator) {\n newActiveIndicator.classList.add(CLASS_NAME_ACTIVE)\n newActiveIndicator.setAttribute('aria-current', 'true')\n }\n }\n\n _updateInterval() {\n const element = this._activeElement || this._getActive()\n\n if (!element) {\n return\n }\n\n const elementInterval = Number.parseInt(element.getAttribute('data-bs-interval'), 10)\n\n this._config.interval = elementInterval || this._config.defaultInterval\n }\n\n _slide(order, element = null) {\n if (this._isSliding) {\n return\n }\n\n const activeElement = this._getActive()\n const isNext = order === ORDER_NEXT\n const nextElement = element || getNextActiveElement(this._getItems(), activeElement, isNext, this._config.wrap)\n\n if (nextElement === activeElement) {\n return\n }\n\n const nextElementIndex = this._getItemIndex(nextElement)\n\n const triggerEvent = eventName => {\n return EventHandler.trigger(this._element, eventName, {\n relatedTarget: nextElement,\n direction: this._orderToDirection(order),\n from: this._getItemIndex(activeElement),\n to: nextElementIndex\n })\n }\n\n const slideEvent = triggerEvent(EVENT_SLIDE)\n\n if (slideEvent.defaultPrevented) {\n return\n }\n\n if (!activeElement || !nextElement) {\n // Some weirdness is happening, so we bail\n // TODO: change tests that use empty divs to avoid this check\n return\n }\n\n const isCycling = Boolean(this._interval)\n this.pause()\n\n this._isSliding = true\n\n this._setActiveIndicatorElement(nextElementIndex)\n this._activeElement = nextElement\n\n const directionalClassName = isNext ? CLASS_NAME_START : CLASS_NAME_END\n const orderClassName = isNext ? CLASS_NAME_NEXT : CLASS_NAME_PREV\n\n nextElement.classList.add(orderClassName)\n\n reflow(nextElement)\n\n activeElement.classList.add(directionalClassName)\n nextElement.classList.add(directionalClassName)\n\n const completeCallBack = () => {\n nextElement.classList.remove(directionalClassName, orderClassName)\n nextElement.classList.add(CLASS_NAME_ACTIVE)\n\n activeElement.classList.remove(CLASS_NAME_ACTIVE, orderClassName, directionalClassName)\n\n this._isSliding = false\n\n triggerEvent(EVENT_SLID)\n }\n\n this._queueCallback(completeCallBack, activeElement, this._isAnimated())\n\n if (isCycling) {\n this.cycle()\n }\n }\n\n _isAnimated() {\n return this._element.classList.contains(CLASS_NAME_SLIDE)\n }\n\n _getActive() {\n return SelectorEngine.findOne(SELECTOR_ACTIVE_ITEM, this._element)\n }\n\n _getItems() {\n return SelectorEngine.find(SELECTOR_ITEM, this._element)\n }\n\n _clearInterval() {\n if (this._interval) {\n clearInterval(this._interval)\n this._interval = null\n }\n }\n\n _directionToOrder(direction) {\n if (isRTL()) {\n return direction === DIRECTION_LEFT ? ORDER_PREV : ORDER_NEXT\n }\n\n return direction === DIRECTION_LEFT ? ORDER_NEXT : ORDER_PREV\n }\n\n _orderToDirection(order) {\n if (isRTL()) {\n return order === ORDER_PREV ? DIRECTION_LEFT : DIRECTION_RIGHT\n }\n\n return order === ORDER_PREV ? DIRECTION_RIGHT : DIRECTION_LEFT\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Carousel.getOrCreateInstance(this, config)\n\n if (typeof config === 'number') {\n data.to(config)\n return\n }\n\n if (typeof config === 'string') {\n if (data[config] === undefined || config.startsWith('_') || config === 'constructor') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config]()\n }\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_SLIDE, function (event) {\n const target = SelectorEngine.getElementFromSelector(this)\n\n if (!target || !target.classList.contains(CLASS_NAME_CAROUSEL)) {\n return\n }\n\n event.preventDefault()\n\n const carousel = Carousel.getOrCreateInstance(target)\n const slideIndex = this.getAttribute('data-bs-slide-to')\n\n if (slideIndex) {\n carousel.to(slideIndex)\n carousel._maybeEnableCycle()\n return\n }\n\n if (Manipulator.getDataAttribute(this, 'slide') === 'next') {\n carousel.next()\n carousel._maybeEnableCycle()\n return\n }\n\n carousel.prev()\n carousel._maybeEnableCycle()\n})\n\nEventHandler.on(window, EVENT_LOAD_DATA_API, () => {\n const carousels = SelectorEngine.find(SELECTOR_DATA_RIDE)\n\n for (const carousel of carousels) {\n Carousel.getOrCreateInstance(carousel)\n }\n})\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Carousel)\n\nexport default Carousel\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap collapse.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport {\n defineJQueryPlugin,\n getElement,\n reflow\n} from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'collapse'\nconst DATA_KEY = 'bs.collapse'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\n\nconst EVENT_SHOW = `show${EVENT_KEY}`\nconst EVENT_SHOWN = `shown${EVENT_KEY}`\nconst EVENT_HIDE = `hide${EVENT_KEY}`\nconst EVENT_HIDDEN = `hidden${EVENT_KEY}`\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\n\nconst CLASS_NAME_SHOW = 'show'\nconst CLASS_NAME_COLLAPSE = 'collapse'\nconst CLASS_NAME_COLLAPSING = 'collapsing'\nconst CLASS_NAME_COLLAPSED = 'collapsed'\nconst CLASS_NAME_DEEPER_CHILDREN = `:scope .${CLASS_NAME_COLLAPSE} .${CLASS_NAME_COLLAPSE}`\nconst CLASS_NAME_HORIZONTAL = 'collapse-horizontal'\n\nconst WIDTH = 'width'\nconst HEIGHT = 'height'\n\nconst SELECTOR_ACTIVES = '.collapse.show, .collapse.collapsing'\nconst SELECTOR_DATA_TOGGLE = '[data-bs-toggle=\"collapse\"]'\n\nconst Default = {\n parent: null,\n toggle: true\n}\n\nconst DefaultType = {\n parent: '(null|element)',\n toggle: 'boolean'\n}\n\n/**\n * Class definition\n */\n\nclass Collapse extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n this._isTransitioning = false\n this._triggerArray = []\n\n const toggleList = SelectorEngine.find(SELECTOR_DATA_TOGGLE)\n\n for (const elem of toggleList) {\n const selector = SelectorEngine.getSelectorFromElement(elem)\n const filterElement = SelectorEngine.find(selector)\n .filter(foundElement => foundElement === this._element)\n\n if (selector !== null && filterElement.length) {\n this._triggerArray.push(elem)\n }\n }\n\n this._initializeChildren()\n\n if (!this._config.parent) {\n this._addAriaAndCollapsedClass(this._triggerArray, this._isShown())\n }\n\n if (this._config.toggle) {\n this.toggle()\n }\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n toggle() {\n if (this._isShown()) {\n this.hide()\n } else {\n this.show()\n }\n }\n\n show() {\n if (this._isTransitioning || this._isShown()) {\n return\n }\n\n let activeChildren = []\n\n // find active children\n if (this._config.parent) {\n activeChildren = this._getFirstLevelChildren(SELECTOR_ACTIVES)\n .filter(element => element !== this._element)\n .map(element => Collapse.getOrCreateInstance(element, { toggle: false }))\n }\n\n if (activeChildren.length && activeChildren[0]._isTransitioning) {\n return\n }\n\n const startEvent = EventHandler.trigger(this._element, EVENT_SHOW)\n if (startEvent.defaultPrevented) {\n return\n }\n\n for (const activeInstance of activeChildren) {\n activeInstance.hide()\n }\n\n const dimension = this._getDimension()\n\n this._element.classList.remove(CLASS_NAME_COLLAPSE)\n this._element.classList.add(CLASS_NAME_COLLAPSING)\n\n this._element.style[dimension] = 0\n\n this._addAriaAndCollapsedClass(this._triggerArray, true)\n this._isTransitioning = true\n\n const complete = () => {\n this._isTransitioning = false\n\n this._element.classList.remove(CLASS_NAME_COLLAPSING)\n this._element.classList.add(CLASS_NAME_COLLAPSE, CLASS_NAME_SHOW)\n\n this._element.style[dimension] = ''\n\n EventHandler.trigger(this._element, EVENT_SHOWN)\n }\n\n const capitalizedDimension = dimension[0].toUpperCase() + dimension.slice(1)\n const scrollSize = `scroll${capitalizedDimension}`\n\n this._queueCallback(complete, this._element, true)\n this._element.style[dimension] = `${this._element[scrollSize]}px`\n }\n\n hide() {\n if (this._isTransitioning || !this._isShown()) {\n return\n }\n\n const startEvent = EventHandler.trigger(this._element, EVENT_HIDE)\n if (startEvent.defaultPrevented) {\n return\n }\n\n const dimension = this._getDimension()\n\n this._element.style[dimension] = `${this._element.getBoundingClientRect()[dimension]}px`\n\n reflow(this._element)\n\n this._element.classList.add(CLASS_NAME_COLLAPSING)\n this._element.classList.remove(CLASS_NAME_COLLAPSE, CLASS_NAME_SHOW)\n\n for (const trigger of this._triggerArray) {\n const element = SelectorEngine.getElementFromSelector(trigger)\n\n if (element && !this._isShown(element)) {\n this._addAriaAndCollapsedClass([trigger], false)\n }\n }\n\n this._isTransitioning = true\n\n const complete = () => {\n this._isTransitioning = false\n this._element.classList.remove(CLASS_NAME_COLLAPSING)\n this._element.classList.add(CLASS_NAME_COLLAPSE)\n EventHandler.trigger(this._element, EVENT_HIDDEN)\n }\n\n this._element.style[dimension] = ''\n\n this._queueCallback(complete, this._element, true)\n }\n\n _isShown(element = this._element) {\n return element.classList.contains(CLASS_NAME_SHOW)\n }\n\n // Private\n _configAfterMerge(config) {\n config.toggle = Boolean(config.toggle) // Coerce string values\n config.parent = getElement(config.parent)\n return config\n }\n\n _getDimension() {\n return this._element.classList.contains(CLASS_NAME_HORIZONTAL) ? WIDTH : HEIGHT\n }\n\n _initializeChildren() {\n if (!this._config.parent) {\n return\n }\n\n const children = this._getFirstLevelChildren(SELECTOR_DATA_TOGGLE)\n\n for (const element of children) {\n const selected = SelectorEngine.getElementFromSelector(element)\n\n if (selected) {\n this._addAriaAndCollapsedClass([element], this._isShown(selected))\n }\n }\n }\n\n _getFirstLevelChildren(selector) {\n const children = SelectorEngine.find(CLASS_NAME_DEEPER_CHILDREN, this._config.parent)\n // remove children if greater depth\n return SelectorEngine.find(selector, this._config.parent).filter(element => !children.includes(element))\n }\n\n _addAriaAndCollapsedClass(triggerArray, isOpen) {\n if (!triggerArray.length) {\n return\n }\n\n for (const element of triggerArray) {\n element.classList.toggle(CLASS_NAME_COLLAPSED, !isOpen)\n element.setAttribute('aria-expanded', isOpen)\n }\n }\n\n // Static\n static jQueryInterface(config) {\n const _config = {}\n if (typeof config === 'string' && /show|hide/.test(config)) {\n _config.toggle = false\n }\n\n return this.each(function () {\n const data = Collapse.getOrCreateInstance(this, _config)\n\n if (typeof config === 'string') {\n if (typeof data[config] === 'undefined') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config]()\n }\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_TOGGLE, function (event) {\n // preventDefault only for elements (which change the URL) not inside the collapsible element\n if (event.target.tagName === 'A' || (event.delegateTarget && event.delegateTarget.tagName === 'A')) {\n event.preventDefault()\n }\n\n for (const element of SelectorEngine.getMultipleElementsFromSelector(this)) {\n Collapse.getOrCreateInstance(element, { toggle: false }).toggle()\n }\n})\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Collapse)\n\nexport default Collapse\n","export var top = 'top';\nexport var bottom = 'bottom';\nexport var right = 'right';\nexport var left = 'left';\nexport var auto = 'auto';\nexport var basePlacements = [top, bottom, right, left];\nexport var start = 'start';\nexport var end = 'end';\nexport var clippingParents = 'clippingParents';\nexport var viewport = 'viewport';\nexport var popper = 'popper';\nexport var reference = 'reference';\nexport var variationPlacements = /*#__PURE__*/basePlacements.reduce(function (acc, placement) {\n return acc.concat([placement + \"-\" + start, placement + \"-\" + end]);\n}, []);\nexport var placements = /*#__PURE__*/[].concat(basePlacements, [auto]).reduce(function (acc, placement) {\n return acc.concat([placement, placement + \"-\" + start, placement + \"-\" + end]);\n}, []); // modifiers that need to read the DOM\n\nexport var beforeRead = 'beforeRead';\nexport var read = 'read';\nexport var afterRead = 'afterRead'; // pure-logic modifiers\n\nexport var beforeMain = 'beforeMain';\nexport var main = 'main';\nexport var afterMain = 'afterMain'; // modifier with the purpose to write to the DOM (or write into a framework state)\n\nexport var beforeWrite = 'beforeWrite';\nexport var write = 'write';\nexport var afterWrite = 'afterWrite';\nexport var modifierPhases = [beforeRead, read, afterRead, beforeMain, main, afterMain, beforeWrite, write, afterWrite];","export default function getNodeName(element) {\n return element ? (element.nodeName || '').toLowerCase() : null;\n}","export default function getWindow(node) {\n if (node == null) {\n return window;\n }\n\n if (node.toString() !== '[object Window]') {\n var ownerDocument = node.ownerDocument;\n return ownerDocument ? ownerDocument.defaultView || window : window;\n }\n\n return node;\n}","import getWindow from \"./getWindow.js\";\n\nfunction isElement(node) {\n var OwnElement = getWindow(node).Element;\n return node instanceof OwnElement || node instanceof Element;\n}\n\nfunction isHTMLElement(node) {\n var OwnElement = getWindow(node).HTMLElement;\n return node instanceof OwnElement || node instanceof HTMLElement;\n}\n\nfunction isShadowRoot(node) {\n // IE 11 has no ShadowRoot\n if (typeof ShadowRoot === 'undefined') {\n return false;\n }\n\n var OwnElement = getWindow(node).ShadowRoot;\n return node instanceof OwnElement || node instanceof ShadowRoot;\n}\n\nexport { isElement, isHTMLElement, isShadowRoot };","import getNodeName from \"../dom-utils/getNodeName.js\";\nimport { isHTMLElement } from \"../dom-utils/instanceOf.js\"; // This modifier takes the styles prepared by the `computeStyles` modifier\n// and applies them to the HTMLElements such as popper and arrow\n\nfunction applyStyles(_ref) {\n var state = _ref.state;\n Object.keys(state.elements).forEach(function (name) {\n var style = state.styles[name] || {};\n var attributes = state.attributes[name] || {};\n var element = state.elements[name]; // arrow is optional + virtual elements\n\n if (!isHTMLElement(element) || !getNodeName(element)) {\n return;\n } // Flow doesn't support to extend this property, but it's the most\n // effective way to apply styles to an HTMLElement\n // $FlowFixMe[cannot-write]\n\n\n Object.assign(element.style, style);\n Object.keys(attributes).forEach(function (name) {\n var value = attributes[name];\n\n if (value === false) {\n element.removeAttribute(name);\n } else {\n element.setAttribute(name, value === true ? '' : value);\n }\n });\n });\n}\n\nfunction effect(_ref2) {\n var state = _ref2.state;\n var initialStyles = {\n popper: {\n position: state.options.strategy,\n left: '0',\n top: '0',\n margin: '0'\n },\n arrow: {\n position: 'absolute'\n },\n reference: {}\n };\n Object.assign(state.elements.popper.style, initialStyles.popper);\n state.styles = initialStyles;\n\n if (state.elements.arrow) {\n Object.assign(state.elements.arrow.style, initialStyles.arrow);\n }\n\n return function () {\n Object.keys(state.elements).forEach(function (name) {\n var element = state.elements[name];\n var attributes = state.attributes[name] || {};\n var styleProperties = Object.keys(state.styles.hasOwnProperty(name) ? state.styles[name] : initialStyles[name]); // Set all values to an empty string to unset them\n\n var style = styleProperties.reduce(function (style, property) {\n style[property] = '';\n return style;\n }, {}); // arrow is optional + virtual elements\n\n if (!isHTMLElement(element) || !getNodeName(element)) {\n return;\n }\n\n Object.assign(element.style, style);\n Object.keys(attributes).forEach(function (attribute) {\n element.removeAttribute(attribute);\n });\n });\n };\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'applyStyles',\n enabled: true,\n phase: 'write',\n fn: applyStyles,\n effect: effect,\n requires: ['computeStyles']\n};","import { auto } from \"../enums.js\";\nexport default function getBasePlacement(placement) {\n return placement.split('-')[0];\n}","export var max = Math.max;\nexport var min = Math.min;\nexport var round = Math.round;","export default function getUAString() {\n var uaData = navigator.userAgentData;\n\n if (uaData != null && uaData.brands && Array.isArray(uaData.brands)) {\n return uaData.brands.map(function (item) {\n return item.brand + \"/\" + item.version;\n }).join(' ');\n }\n\n return navigator.userAgent;\n}","import getUAString from \"../utils/userAgent.js\";\nexport default function isLayoutViewport() {\n return !/^((?!chrome|android).)*safari/i.test(getUAString());\n}","import { isElement, isHTMLElement } from \"./instanceOf.js\";\nimport { round } from \"../utils/math.js\";\nimport getWindow from \"./getWindow.js\";\nimport isLayoutViewport from \"./isLayoutViewport.js\";\nexport default function getBoundingClientRect(element, includeScale, isFixedStrategy) {\n if (includeScale === void 0) {\n includeScale = false;\n }\n\n if (isFixedStrategy === void 0) {\n isFixedStrategy = false;\n }\n\n var clientRect = element.getBoundingClientRect();\n var scaleX = 1;\n var scaleY = 1;\n\n if (includeScale && isHTMLElement(element)) {\n scaleX = element.offsetWidth > 0 ? round(clientRect.width) / element.offsetWidth || 1 : 1;\n scaleY = element.offsetHeight > 0 ? round(clientRect.height) / element.offsetHeight || 1 : 1;\n }\n\n var _ref = isElement(element) ? getWindow(element) : window,\n visualViewport = _ref.visualViewport;\n\n var addVisualOffsets = !isLayoutViewport() && isFixedStrategy;\n var x = (clientRect.left + (addVisualOffsets && visualViewport ? visualViewport.offsetLeft : 0)) / scaleX;\n var y = (clientRect.top + (addVisualOffsets && visualViewport ? visualViewport.offsetTop : 0)) / scaleY;\n var width = clientRect.width / scaleX;\n var height = clientRect.height / scaleY;\n return {\n width: width,\n height: height,\n top: y,\n right: x + width,\n bottom: y + height,\n left: x,\n x: x,\n y: y\n };\n}","import getBoundingClientRect from \"./getBoundingClientRect.js\"; // Returns the layout rect of an element relative to its offsetParent. Layout\n// means it doesn't take into account transforms.\n\nexport default function getLayoutRect(element) {\n var clientRect = getBoundingClientRect(element); // Use the clientRect sizes if it's not been transformed.\n // Fixes https://github.com/popperjs/popper-core/issues/1223\n\n var width = element.offsetWidth;\n var height = element.offsetHeight;\n\n if (Math.abs(clientRect.width - width) <= 1) {\n width = clientRect.width;\n }\n\n if (Math.abs(clientRect.height - height) <= 1) {\n height = clientRect.height;\n }\n\n return {\n x: element.offsetLeft,\n y: element.offsetTop,\n width: width,\n height: height\n };\n}","import { isShadowRoot } from \"./instanceOf.js\";\nexport default function contains(parent, child) {\n var rootNode = child.getRootNode && child.getRootNode(); // First, attempt with faster native method\n\n if (parent.contains(child)) {\n return true;\n } // then fallback to custom implementation with Shadow DOM support\n else if (rootNode && isShadowRoot(rootNode)) {\n var next = child;\n\n do {\n if (next && parent.isSameNode(next)) {\n return true;\n } // $FlowFixMe[prop-missing]: need a better way to handle this...\n\n\n next = next.parentNode || next.host;\n } while (next);\n } // Give up, the result is false\n\n\n return false;\n}","import getWindow from \"./getWindow.js\";\nexport default function getComputedStyle(element) {\n return getWindow(element).getComputedStyle(element);\n}","import getNodeName from \"./getNodeName.js\";\nexport default function isTableElement(element) {\n return ['table', 'td', 'th'].indexOf(getNodeName(element)) >= 0;\n}","import { isElement } from \"./instanceOf.js\";\nexport default function getDocumentElement(element) {\n // $FlowFixMe[incompatible-return]: assume body is always available\n return ((isElement(element) ? element.ownerDocument : // $FlowFixMe[prop-missing]\n element.document) || window.document).documentElement;\n}","import getNodeName from \"./getNodeName.js\";\nimport getDocumentElement from \"./getDocumentElement.js\";\nimport { isShadowRoot } from \"./instanceOf.js\";\nexport default function getParentNode(element) {\n if (getNodeName(element) === 'html') {\n return element;\n }\n\n return (// this is a quicker (but less type safe) way to save quite some bytes from the bundle\n // $FlowFixMe[incompatible-return]\n // $FlowFixMe[prop-missing]\n element.assignedSlot || // step into the shadow DOM of the parent of a slotted node\n element.parentNode || ( // DOM Element detected\n isShadowRoot(element) ? element.host : null) || // ShadowRoot detected\n // $FlowFixMe[incompatible-call]: HTMLElement is a Node\n getDocumentElement(element) // fallback\n\n );\n}","import getWindow from \"./getWindow.js\";\nimport getNodeName from \"./getNodeName.js\";\nimport getComputedStyle from \"./getComputedStyle.js\";\nimport { isHTMLElement, isShadowRoot } from \"./instanceOf.js\";\nimport isTableElement from \"./isTableElement.js\";\nimport getParentNode from \"./getParentNode.js\";\nimport getUAString from \"../utils/userAgent.js\";\n\nfunction getTrueOffsetParent(element) {\n if (!isHTMLElement(element) || // https://github.com/popperjs/popper-core/issues/837\n getComputedStyle(element).position === 'fixed') {\n return null;\n }\n\n return element.offsetParent;\n} // `.offsetParent` reports `null` for fixed elements, while absolute elements\n// return the containing block\n\n\nfunction getContainingBlock(element) {\n var isFirefox = /firefox/i.test(getUAString());\n var isIE = /Trident/i.test(getUAString());\n\n if (isIE && isHTMLElement(element)) {\n // In IE 9, 10 and 11 fixed elements containing block is always established by the viewport\n var elementCss = getComputedStyle(element);\n\n if (elementCss.position === 'fixed') {\n return null;\n }\n }\n\n var currentNode = getParentNode(element);\n\n if (isShadowRoot(currentNode)) {\n currentNode = currentNode.host;\n }\n\n while (isHTMLElement(currentNode) && ['html', 'body'].indexOf(getNodeName(currentNode)) < 0) {\n var css = getComputedStyle(currentNode); // This is non-exhaustive but covers the most common CSS properties that\n // create a containing block.\n // https://developer.mozilla.org/en-US/docs/Web/CSS/Containing_block#identifying_the_containing_block\n\n if (css.transform !== 'none' || css.perspective !== 'none' || css.contain === 'paint' || ['transform', 'perspective'].indexOf(css.willChange) !== -1 || isFirefox && css.willChange === 'filter' || isFirefox && css.filter && css.filter !== 'none') {\n return currentNode;\n } else {\n currentNode = currentNode.parentNode;\n }\n }\n\n return null;\n} // Gets the closest ancestor positioned element. Handles some edge cases,\n// such as table ancestors and cross browser bugs.\n\n\nexport default function getOffsetParent(element) {\n var window = getWindow(element);\n var offsetParent = getTrueOffsetParent(element);\n\n while (offsetParent && isTableElement(offsetParent) && getComputedStyle(offsetParent).position === 'static') {\n offsetParent = getTrueOffsetParent(offsetParent);\n }\n\n if (offsetParent && (getNodeName(offsetParent) === 'html' || getNodeName(offsetParent) === 'body' && getComputedStyle(offsetParent).position === 'static')) {\n return window;\n }\n\n return offsetParent || getContainingBlock(element) || window;\n}","export default function getMainAxisFromPlacement(placement) {\n return ['top', 'bottom'].indexOf(placement) >= 0 ? 'x' : 'y';\n}","import { max as mathMax, min as mathMin } from \"./math.js\";\nexport function within(min, value, max) {\n return mathMax(min, mathMin(value, max));\n}\nexport function withinMaxClamp(min, value, max) {\n var v = within(min, value, max);\n return v > max ? max : v;\n}","import getFreshSideObject from \"./getFreshSideObject.js\";\nexport default function mergePaddingObject(paddingObject) {\n return Object.assign({}, getFreshSideObject(), paddingObject);\n}","export default function getFreshSideObject() {\n return {\n top: 0,\n right: 0,\n bottom: 0,\n left: 0\n };\n}","export default function expandToHashMap(value, keys) {\n return keys.reduce(function (hashMap, key) {\n hashMap[key] = value;\n return hashMap;\n }, {});\n}","import getBasePlacement from \"../utils/getBasePlacement.js\";\nimport getLayoutRect from \"../dom-utils/getLayoutRect.js\";\nimport contains from \"../dom-utils/contains.js\";\nimport getOffsetParent from \"../dom-utils/getOffsetParent.js\";\nimport getMainAxisFromPlacement from \"../utils/getMainAxisFromPlacement.js\";\nimport { within } from \"../utils/within.js\";\nimport mergePaddingObject from \"../utils/mergePaddingObject.js\";\nimport expandToHashMap from \"../utils/expandToHashMap.js\";\nimport { left, right, basePlacements, top, bottom } from \"../enums.js\"; // eslint-disable-next-line import/no-unused-modules\n\nvar toPaddingObject = function toPaddingObject(padding, state) {\n padding = typeof padding === 'function' ? padding(Object.assign({}, state.rects, {\n placement: state.placement\n })) : padding;\n return mergePaddingObject(typeof padding !== 'number' ? padding : expandToHashMap(padding, basePlacements));\n};\n\nfunction arrow(_ref) {\n var _state$modifiersData$;\n\n var state = _ref.state,\n name = _ref.name,\n options = _ref.options;\n var arrowElement = state.elements.arrow;\n var popperOffsets = state.modifiersData.popperOffsets;\n var basePlacement = getBasePlacement(state.placement);\n var axis = getMainAxisFromPlacement(basePlacement);\n var isVertical = [left, right].indexOf(basePlacement) >= 0;\n var len = isVertical ? 'height' : 'width';\n\n if (!arrowElement || !popperOffsets) {\n return;\n }\n\n var paddingObject = toPaddingObject(options.padding, state);\n var arrowRect = getLayoutRect(arrowElement);\n var minProp = axis === 'y' ? top : left;\n var maxProp = axis === 'y' ? bottom : right;\n var endDiff = state.rects.reference[len] + state.rects.reference[axis] - popperOffsets[axis] - state.rects.popper[len];\n var startDiff = popperOffsets[axis] - state.rects.reference[axis];\n var arrowOffsetParent = getOffsetParent(arrowElement);\n var clientSize = arrowOffsetParent ? axis === 'y' ? arrowOffsetParent.clientHeight || 0 : arrowOffsetParent.clientWidth || 0 : 0;\n var centerToReference = endDiff / 2 - startDiff / 2; // Make sure the arrow doesn't overflow the popper if the center point is\n // outside of the popper bounds\n\n var min = paddingObject[minProp];\n var max = clientSize - arrowRect[len] - paddingObject[maxProp];\n var center = clientSize / 2 - arrowRect[len] / 2 + centerToReference;\n var offset = within(min, center, max); // Prevents breaking syntax highlighting...\n\n var axisProp = axis;\n state.modifiersData[name] = (_state$modifiersData$ = {}, _state$modifiersData$[axisProp] = offset, _state$modifiersData$.centerOffset = offset - center, _state$modifiersData$);\n}\n\nfunction effect(_ref2) {\n var state = _ref2.state,\n options = _ref2.options;\n var _options$element = options.element,\n arrowElement = _options$element === void 0 ? '[data-popper-arrow]' : _options$element;\n\n if (arrowElement == null) {\n return;\n } // CSS selector\n\n\n if (typeof arrowElement === 'string') {\n arrowElement = state.elements.popper.querySelector(arrowElement);\n\n if (!arrowElement) {\n return;\n }\n }\n\n if (!contains(state.elements.popper, arrowElement)) {\n return;\n }\n\n state.elements.arrow = arrowElement;\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'arrow',\n enabled: true,\n phase: 'main',\n fn: arrow,\n effect: effect,\n requires: ['popperOffsets'],\n requiresIfExists: ['preventOverflow']\n};","export default function getVariation(placement) {\n return placement.split('-')[1];\n}","import { top, left, right, bottom, end } from \"../enums.js\";\nimport getOffsetParent from \"../dom-utils/getOffsetParent.js\";\nimport getWindow from \"../dom-utils/getWindow.js\";\nimport getDocumentElement from \"../dom-utils/getDocumentElement.js\";\nimport getComputedStyle from \"../dom-utils/getComputedStyle.js\";\nimport getBasePlacement from \"../utils/getBasePlacement.js\";\nimport getVariation from \"../utils/getVariation.js\";\nimport { round } from \"../utils/math.js\"; // eslint-disable-next-line import/no-unused-modules\n\nvar unsetSides = {\n top: 'auto',\n right: 'auto',\n bottom: 'auto',\n left: 'auto'\n}; // Round the offsets to the nearest suitable subpixel based on the DPR.\n// Zooming can change the DPR, but it seems to report a value that will\n// cleanly divide the values into the appropriate subpixels.\n\nfunction roundOffsetsByDPR(_ref, win) {\n var x = _ref.x,\n y = _ref.y;\n var dpr = win.devicePixelRatio || 1;\n return {\n x: round(x * dpr) / dpr || 0,\n y: round(y * dpr) / dpr || 0\n };\n}\n\nexport function mapToStyles(_ref2) {\n var _Object$assign2;\n\n var popper = _ref2.popper,\n popperRect = _ref2.popperRect,\n placement = _ref2.placement,\n variation = _ref2.variation,\n offsets = _ref2.offsets,\n position = _ref2.position,\n gpuAcceleration = _ref2.gpuAcceleration,\n adaptive = _ref2.adaptive,\n roundOffsets = _ref2.roundOffsets,\n isFixed = _ref2.isFixed;\n var _offsets$x = offsets.x,\n x = _offsets$x === void 0 ? 0 : _offsets$x,\n _offsets$y = offsets.y,\n y = _offsets$y === void 0 ? 0 : _offsets$y;\n\n var _ref3 = typeof roundOffsets === 'function' ? roundOffsets({\n x: x,\n y: y\n }) : {\n x: x,\n y: y\n };\n\n x = _ref3.x;\n y = _ref3.y;\n var hasX = offsets.hasOwnProperty('x');\n var hasY = offsets.hasOwnProperty('y');\n var sideX = left;\n var sideY = top;\n var win = window;\n\n if (adaptive) {\n var offsetParent = getOffsetParent(popper);\n var heightProp = 'clientHeight';\n var widthProp = 'clientWidth';\n\n if (offsetParent === getWindow(popper)) {\n offsetParent = getDocumentElement(popper);\n\n if (getComputedStyle(offsetParent).position !== 'static' && position === 'absolute') {\n heightProp = 'scrollHeight';\n widthProp = 'scrollWidth';\n }\n } // $FlowFixMe[incompatible-cast]: force type refinement, we compare offsetParent with window above, but Flow doesn't detect it\n\n\n offsetParent = offsetParent;\n\n if (placement === top || (placement === left || placement === right) && variation === end) {\n sideY = bottom;\n var offsetY = isFixed && offsetParent === win && win.visualViewport ? win.visualViewport.height : // $FlowFixMe[prop-missing]\n offsetParent[heightProp];\n y -= offsetY - popperRect.height;\n y *= gpuAcceleration ? 1 : -1;\n }\n\n if (placement === left || (placement === top || placement === bottom) && variation === end) {\n sideX = right;\n var offsetX = isFixed && offsetParent === win && win.visualViewport ? win.visualViewport.width : // $FlowFixMe[prop-missing]\n offsetParent[widthProp];\n x -= offsetX - popperRect.width;\n x *= gpuAcceleration ? 1 : -1;\n }\n }\n\n var commonStyles = Object.assign({\n position: position\n }, adaptive && unsetSides);\n\n var _ref4 = roundOffsets === true ? roundOffsetsByDPR({\n x: x,\n y: y\n }, getWindow(popper)) : {\n x: x,\n y: y\n };\n\n x = _ref4.x;\n y = _ref4.y;\n\n if (gpuAcceleration) {\n var _Object$assign;\n\n return Object.assign({}, commonStyles, (_Object$assign = {}, _Object$assign[sideY] = hasY ? '0' : '', _Object$assign[sideX] = hasX ? '0' : '', _Object$assign.transform = (win.devicePixelRatio || 1) <= 1 ? \"translate(\" + x + \"px, \" + y + \"px)\" : \"translate3d(\" + x + \"px, \" + y + \"px, 0)\", _Object$assign));\n }\n\n return Object.assign({}, commonStyles, (_Object$assign2 = {}, _Object$assign2[sideY] = hasY ? y + \"px\" : '', _Object$assign2[sideX] = hasX ? x + \"px\" : '', _Object$assign2.transform = '', _Object$assign2));\n}\n\nfunction computeStyles(_ref5) {\n var state = _ref5.state,\n options = _ref5.options;\n var _options$gpuAccelerat = options.gpuAcceleration,\n gpuAcceleration = _options$gpuAccelerat === void 0 ? true : _options$gpuAccelerat,\n _options$adaptive = options.adaptive,\n adaptive = _options$adaptive === void 0 ? true : _options$adaptive,\n _options$roundOffsets = options.roundOffsets,\n roundOffsets = _options$roundOffsets === void 0 ? true : _options$roundOffsets;\n var commonStyles = {\n placement: getBasePlacement(state.placement),\n variation: getVariation(state.placement),\n popper: state.elements.popper,\n popperRect: state.rects.popper,\n gpuAcceleration: gpuAcceleration,\n isFixed: state.options.strategy === 'fixed'\n };\n\n if (state.modifiersData.popperOffsets != null) {\n state.styles.popper = Object.assign({}, state.styles.popper, mapToStyles(Object.assign({}, commonStyles, {\n offsets: state.modifiersData.popperOffsets,\n position: state.options.strategy,\n adaptive: adaptive,\n roundOffsets: roundOffsets\n })));\n }\n\n if (state.modifiersData.arrow != null) {\n state.styles.arrow = Object.assign({}, state.styles.arrow, mapToStyles(Object.assign({}, commonStyles, {\n offsets: state.modifiersData.arrow,\n position: 'absolute',\n adaptive: false,\n roundOffsets: roundOffsets\n })));\n }\n\n state.attributes.popper = Object.assign({}, state.attributes.popper, {\n 'data-popper-placement': state.placement\n });\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'computeStyles',\n enabled: true,\n phase: 'beforeWrite',\n fn: computeStyles,\n data: {}\n};","import getWindow from \"../dom-utils/getWindow.js\"; // eslint-disable-next-line import/no-unused-modules\n\nvar passive = {\n passive: true\n};\n\nfunction effect(_ref) {\n var state = _ref.state,\n instance = _ref.instance,\n options = _ref.options;\n var _options$scroll = options.scroll,\n scroll = _options$scroll === void 0 ? true : _options$scroll,\n _options$resize = options.resize,\n resize = _options$resize === void 0 ? true : _options$resize;\n var window = getWindow(state.elements.popper);\n var scrollParents = [].concat(state.scrollParents.reference, state.scrollParents.popper);\n\n if (scroll) {\n scrollParents.forEach(function (scrollParent) {\n scrollParent.addEventListener('scroll', instance.update, passive);\n });\n }\n\n if (resize) {\n window.addEventListener('resize', instance.update, passive);\n }\n\n return function () {\n if (scroll) {\n scrollParents.forEach(function (scrollParent) {\n scrollParent.removeEventListener('scroll', instance.update, passive);\n });\n }\n\n if (resize) {\n window.removeEventListener('resize', instance.update, passive);\n }\n };\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'eventListeners',\n enabled: true,\n phase: 'write',\n fn: function fn() {},\n effect: effect,\n data: {}\n};","var hash = {\n left: 'right',\n right: 'left',\n bottom: 'top',\n top: 'bottom'\n};\nexport default function getOppositePlacement(placement) {\n return placement.replace(/left|right|bottom|top/g, function (matched) {\n return hash[matched];\n });\n}","var hash = {\n start: 'end',\n end: 'start'\n};\nexport default function getOppositeVariationPlacement(placement) {\n return placement.replace(/start|end/g, function (matched) {\n return hash[matched];\n });\n}","import getWindow from \"./getWindow.js\";\nexport default function getWindowScroll(node) {\n var win = getWindow(node);\n var scrollLeft = win.pageXOffset;\n var scrollTop = win.pageYOffset;\n return {\n scrollLeft: scrollLeft,\n scrollTop: scrollTop\n };\n}","import getBoundingClientRect from \"./getBoundingClientRect.js\";\nimport getDocumentElement from \"./getDocumentElement.js\";\nimport getWindowScroll from \"./getWindowScroll.js\";\nexport default function getWindowScrollBarX(element) {\n // If has a CSS width greater than the viewport, then this will be\n // incorrect for RTL.\n // Popper 1 is broken in this case and never had a bug report so let's assume\n // it's not an issue. I don't think anyone ever specifies width on \n // anyway.\n // Browsers where the left scrollbar doesn't cause an issue report `0` for\n // this (e.g. Edge 2019, IE11, Safari)\n return getBoundingClientRect(getDocumentElement(element)).left + getWindowScroll(element).scrollLeft;\n}","import getComputedStyle from \"./getComputedStyle.js\";\nexport default function isScrollParent(element) {\n // Firefox wants us to check `-x` and `-y` variations as well\n var _getComputedStyle = getComputedStyle(element),\n overflow = _getComputedStyle.overflow,\n overflowX = _getComputedStyle.overflowX,\n overflowY = _getComputedStyle.overflowY;\n\n return /auto|scroll|overlay|hidden/.test(overflow + overflowY + overflowX);\n}","import getParentNode from \"./getParentNode.js\";\nimport isScrollParent from \"./isScrollParent.js\";\nimport getNodeName from \"./getNodeName.js\";\nimport { isHTMLElement } from \"./instanceOf.js\";\nexport default function getScrollParent(node) {\n if (['html', 'body', '#document'].indexOf(getNodeName(node)) >= 0) {\n // $FlowFixMe[incompatible-return]: assume body is always available\n return node.ownerDocument.body;\n }\n\n if (isHTMLElement(node) && isScrollParent(node)) {\n return node;\n }\n\n return getScrollParent(getParentNode(node));\n}","import getScrollParent from \"./getScrollParent.js\";\nimport getParentNode from \"./getParentNode.js\";\nimport getWindow from \"./getWindow.js\";\nimport isScrollParent from \"./isScrollParent.js\";\n/*\ngiven a DOM element, return the list of all scroll parents, up the list of ancesors\nuntil we get to the top window object. This list is what we attach scroll listeners\nto, because if any of these parent elements scroll, we'll need to re-calculate the\nreference element's position.\n*/\n\nexport default function listScrollParents(element, list) {\n var _element$ownerDocumen;\n\n if (list === void 0) {\n list = [];\n }\n\n var scrollParent = getScrollParent(element);\n var isBody = scrollParent === ((_element$ownerDocumen = element.ownerDocument) == null ? void 0 : _element$ownerDocumen.body);\n var win = getWindow(scrollParent);\n var target = isBody ? [win].concat(win.visualViewport || [], isScrollParent(scrollParent) ? scrollParent : []) : scrollParent;\n var updatedList = list.concat(target);\n return isBody ? updatedList : // $FlowFixMe[incompatible-call]: isBody tells us target will be an HTMLElement here\n updatedList.concat(listScrollParents(getParentNode(target)));\n}","export default function rectToClientRect(rect) {\n return Object.assign({}, rect, {\n left: rect.x,\n top: rect.y,\n right: rect.x + rect.width,\n bottom: rect.y + rect.height\n });\n}","import { viewport } from \"../enums.js\";\nimport getViewportRect from \"./getViewportRect.js\";\nimport getDocumentRect from \"./getDocumentRect.js\";\nimport listScrollParents from \"./listScrollParents.js\";\nimport getOffsetParent from \"./getOffsetParent.js\";\nimport getDocumentElement from \"./getDocumentElement.js\";\nimport getComputedStyle from \"./getComputedStyle.js\";\nimport { isElement, isHTMLElement } from \"./instanceOf.js\";\nimport getBoundingClientRect from \"./getBoundingClientRect.js\";\nimport getParentNode from \"./getParentNode.js\";\nimport contains from \"./contains.js\";\nimport getNodeName from \"./getNodeName.js\";\nimport rectToClientRect from \"../utils/rectToClientRect.js\";\nimport { max, min } from \"../utils/math.js\";\n\nfunction getInnerBoundingClientRect(element, strategy) {\n var rect = getBoundingClientRect(element, false, strategy === 'fixed');\n rect.top = rect.top + element.clientTop;\n rect.left = rect.left + element.clientLeft;\n rect.bottom = rect.top + element.clientHeight;\n rect.right = rect.left + element.clientWidth;\n rect.width = element.clientWidth;\n rect.height = element.clientHeight;\n rect.x = rect.left;\n rect.y = rect.top;\n return rect;\n}\n\nfunction getClientRectFromMixedType(element, clippingParent, strategy) {\n return clippingParent === viewport ? rectToClientRect(getViewportRect(element, strategy)) : isElement(clippingParent) ? getInnerBoundingClientRect(clippingParent, strategy) : rectToClientRect(getDocumentRect(getDocumentElement(element)));\n} // A \"clipping parent\" is an overflowable container with the characteristic of\n// clipping (or hiding) overflowing elements with a position different from\n// `initial`\n\n\nfunction getClippingParents(element) {\n var clippingParents = listScrollParents(getParentNode(element));\n var canEscapeClipping = ['absolute', 'fixed'].indexOf(getComputedStyle(element).position) >= 0;\n var clipperElement = canEscapeClipping && isHTMLElement(element) ? getOffsetParent(element) : element;\n\n if (!isElement(clipperElement)) {\n return [];\n } // $FlowFixMe[incompatible-return]: https://github.com/facebook/flow/issues/1414\n\n\n return clippingParents.filter(function (clippingParent) {\n return isElement(clippingParent) && contains(clippingParent, clipperElement) && getNodeName(clippingParent) !== 'body';\n });\n} // Gets the maximum area that the element is visible in due to any number of\n// clipping parents\n\n\nexport default function getClippingRect(element, boundary, rootBoundary, strategy) {\n var mainClippingParents = boundary === 'clippingParents' ? getClippingParents(element) : [].concat(boundary);\n var clippingParents = [].concat(mainClippingParents, [rootBoundary]);\n var firstClippingParent = clippingParents[0];\n var clippingRect = clippingParents.reduce(function (accRect, clippingParent) {\n var rect = getClientRectFromMixedType(element, clippingParent, strategy);\n accRect.top = max(rect.top, accRect.top);\n accRect.right = min(rect.right, accRect.right);\n accRect.bottom = min(rect.bottom, accRect.bottom);\n accRect.left = max(rect.left, accRect.left);\n return accRect;\n }, getClientRectFromMixedType(element, firstClippingParent, strategy));\n clippingRect.width = clippingRect.right - clippingRect.left;\n clippingRect.height = clippingRect.bottom - clippingRect.top;\n clippingRect.x = clippingRect.left;\n clippingRect.y = clippingRect.top;\n return clippingRect;\n}","import getWindow from \"./getWindow.js\";\nimport getDocumentElement from \"./getDocumentElement.js\";\nimport getWindowScrollBarX from \"./getWindowScrollBarX.js\";\nimport isLayoutViewport from \"./isLayoutViewport.js\";\nexport default function getViewportRect(element, strategy) {\n var win = getWindow(element);\n var html = getDocumentElement(element);\n var visualViewport = win.visualViewport;\n var width = html.clientWidth;\n var height = html.clientHeight;\n var x = 0;\n var y = 0;\n\n if (visualViewport) {\n width = visualViewport.width;\n height = visualViewport.height;\n var layoutViewport = isLayoutViewport();\n\n if (layoutViewport || !layoutViewport && strategy === 'fixed') {\n x = visualViewport.offsetLeft;\n y = visualViewport.offsetTop;\n }\n }\n\n return {\n width: width,\n height: height,\n x: x + getWindowScrollBarX(element),\n y: y\n };\n}","import getDocumentElement from \"./getDocumentElement.js\";\nimport getComputedStyle from \"./getComputedStyle.js\";\nimport getWindowScrollBarX from \"./getWindowScrollBarX.js\";\nimport getWindowScroll from \"./getWindowScroll.js\";\nimport { max } from \"../utils/math.js\"; // Gets the entire size of the scrollable document area, even extending outside\n// of the `` and `` rect bounds if horizontally scrollable\n\nexport default function getDocumentRect(element) {\n var _element$ownerDocumen;\n\n var html = getDocumentElement(element);\n var winScroll = getWindowScroll(element);\n var body = (_element$ownerDocumen = element.ownerDocument) == null ? void 0 : _element$ownerDocumen.body;\n var width = max(html.scrollWidth, html.clientWidth, body ? body.scrollWidth : 0, body ? body.clientWidth : 0);\n var height = max(html.scrollHeight, html.clientHeight, body ? body.scrollHeight : 0, body ? body.clientHeight : 0);\n var x = -winScroll.scrollLeft + getWindowScrollBarX(element);\n var y = -winScroll.scrollTop;\n\n if (getComputedStyle(body || html).direction === 'rtl') {\n x += max(html.clientWidth, body ? body.clientWidth : 0) - width;\n }\n\n return {\n width: width,\n height: height,\n x: x,\n y: y\n };\n}","import getBasePlacement from \"./getBasePlacement.js\";\nimport getVariation from \"./getVariation.js\";\nimport getMainAxisFromPlacement from \"./getMainAxisFromPlacement.js\";\nimport { top, right, bottom, left, start, end } from \"../enums.js\";\nexport default function computeOffsets(_ref) {\n var reference = _ref.reference,\n element = _ref.element,\n placement = _ref.placement;\n var basePlacement = placement ? getBasePlacement(placement) : null;\n var variation = placement ? getVariation(placement) : null;\n var commonX = reference.x + reference.width / 2 - element.width / 2;\n var commonY = reference.y + reference.height / 2 - element.height / 2;\n var offsets;\n\n switch (basePlacement) {\n case top:\n offsets = {\n x: commonX,\n y: reference.y - element.height\n };\n break;\n\n case bottom:\n offsets = {\n x: commonX,\n y: reference.y + reference.height\n };\n break;\n\n case right:\n offsets = {\n x: reference.x + reference.width,\n y: commonY\n };\n break;\n\n case left:\n offsets = {\n x: reference.x - element.width,\n y: commonY\n };\n break;\n\n default:\n offsets = {\n x: reference.x,\n y: reference.y\n };\n }\n\n var mainAxis = basePlacement ? getMainAxisFromPlacement(basePlacement) : null;\n\n if (mainAxis != null) {\n var len = mainAxis === 'y' ? 'height' : 'width';\n\n switch (variation) {\n case start:\n offsets[mainAxis] = offsets[mainAxis] - (reference[len] / 2 - element[len] / 2);\n break;\n\n case end:\n offsets[mainAxis] = offsets[mainAxis] + (reference[len] / 2 - element[len] / 2);\n break;\n\n default:\n }\n }\n\n return offsets;\n}","import getClippingRect from \"../dom-utils/getClippingRect.js\";\nimport getDocumentElement from \"../dom-utils/getDocumentElement.js\";\nimport getBoundingClientRect from \"../dom-utils/getBoundingClientRect.js\";\nimport computeOffsets from \"./computeOffsets.js\";\nimport rectToClientRect from \"./rectToClientRect.js\";\nimport { clippingParents, reference, popper, bottom, top, right, basePlacements, viewport } from \"../enums.js\";\nimport { isElement } from \"../dom-utils/instanceOf.js\";\nimport mergePaddingObject from \"./mergePaddingObject.js\";\nimport expandToHashMap from \"./expandToHashMap.js\"; // eslint-disable-next-line import/no-unused-modules\n\nexport default function detectOverflow(state, options) {\n if (options === void 0) {\n options = {};\n }\n\n var _options = options,\n _options$placement = _options.placement,\n placement = _options$placement === void 0 ? state.placement : _options$placement,\n _options$strategy = _options.strategy,\n strategy = _options$strategy === void 0 ? state.strategy : _options$strategy,\n _options$boundary = _options.boundary,\n boundary = _options$boundary === void 0 ? clippingParents : _options$boundary,\n _options$rootBoundary = _options.rootBoundary,\n rootBoundary = _options$rootBoundary === void 0 ? viewport : _options$rootBoundary,\n _options$elementConte = _options.elementContext,\n elementContext = _options$elementConte === void 0 ? popper : _options$elementConte,\n _options$altBoundary = _options.altBoundary,\n altBoundary = _options$altBoundary === void 0 ? false : _options$altBoundary,\n _options$padding = _options.padding,\n padding = _options$padding === void 0 ? 0 : _options$padding;\n var paddingObject = mergePaddingObject(typeof padding !== 'number' ? padding : expandToHashMap(padding, basePlacements));\n var altContext = elementContext === popper ? reference : popper;\n var popperRect = state.rects.popper;\n var element = state.elements[altBoundary ? altContext : elementContext];\n var clippingClientRect = getClippingRect(isElement(element) ? element : element.contextElement || getDocumentElement(state.elements.popper), boundary, rootBoundary, strategy);\n var referenceClientRect = getBoundingClientRect(state.elements.reference);\n var popperOffsets = computeOffsets({\n reference: referenceClientRect,\n element: popperRect,\n strategy: 'absolute',\n placement: placement\n });\n var popperClientRect = rectToClientRect(Object.assign({}, popperRect, popperOffsets));\n var elementClientRect = elementContext === popper ? popperClientRect : referenceClientRect; // positive = overflowing the clipping rect\n // 0 or negative = within the clipping rect\n\n var overflowOffsets = {\n top: clippingClientRect.top - elementClientRect.top + paddingObject.top,\n bottom: elementClientRect.bottom - clippingClientRect.bottom + paddingObject.bottom,\n left: clippingClientRect.left - elementClientRect.left + paddingObject.left,\n right: elementClientRect.right - clippingClientRect.right + paddingObject.right\n };\n var offsetData = state.modifiersData.offset; // Offsets can be applied only to the popper element\n\n if (elementContext === popper && offsetData) {\n var offset = offsetData[placement];\n Object.keys(overflowOffsets).forEach(function (key) {\n var multiply = [right, bottom].indexOf(key) >= 0 ? 1 : -1;\n var axis = [top, bottom].indexOf(key) >= 0 ? 'y' : 'x';\n overflowOffsets[key] += offset[axis] * multiply;\n });\n }\n\n return overflowOffsets;\n}","import getVariation from \"./getVariation.js\";\nimport { variationPlacements, basePlacements, placements as allPlacements } from \"../enums.js\";\nimport detectOverflow from \"./detectOverflow.js\";\nimport getBasePlacement from \"./getBasePlacement.js\";\nexport default function computeAutoPlacement(state, options) {\n if (options === void 0) {\n options = {};\n }\n\n var _options = options,\n placement = _options.placement,\n boundary = _options.boundary,\n rootBoundary = _options.rootBoundary,\n padding = _options.padding,\n flipVariations = _options.flipVariations,\n _options$allowedAutoP = _options.allowedAutoPlacements,\n allowedAutoPlacements = _options$allowedAutoP === void 0 ? allPlacements : _options$allowedAutoP;\n var variation = getVariation(placement);\n var placements = variation ? flipVariations ? variationPlacements : variationPlacements.filter(function (placement) {\n return getVariation(placement) === variation;\n }) : basePlacements;\n var allowedPlacements = placements.filter(function (placement) {\n return allowedAutoPlacements.indexOf(placement) >= 0;\n });\n\n if (allowedPlacements.length === 0) {\n allowedPlacements = placements;\n } // $FlowFixMe[incompatible-type]: Flow seems to have problems with two array unions...\n\n\n var overflows = allowedPlacements.reduce(function (acc, placement) {\n acc[placement] = detectOverflow(state, {\n placement: placement,\n boundary: boundary,\n rootBoundary: rootBoundary,\n padding: padding\n })[getBasePlacement(placement)];\n return acc;\n }, {});\n return Object.keys(overflows).sort(function (a, b) {\n return overflows[a] - overflows[b];\n });\n}","import getOppositePlacement from \"../utils/getOppositePlacement.js\";\nimport getBasePlacement from \"../utils/getBasePlacement.js\";\nimport getOppositeVariationPlacement from \"../utils/getOppositeVariationPlacement.js\";\nimport detectOverflow from \"../utils/detectOverflow.js\";\nimport computeAutoPlacement from \"../utils/computeAutoPlacement.js\";\nimport { bottom, top, start, right, left, auto } from \"../enums.js\";\nimport getVariation from \"../utils/getVariation.js\"; // eslint-disable-next-line import/no-unused-modules\n\nfunction getExpandedFallbackPlacements(placement) {\n if (getBasePlacement(placement) === auto) {\n return [];\n }\n\n var oppositePlacement = getOppositePlacement(placement);\n return [getOppositeVariationPlacement(placement), oppositePlacement, getOppositeVariationPlacement(oppositePlacement)];\n}\n\nfunction flip(_ref) {\n var state = _ref.state,\n options = _ref.options,\n name = _ref.name;\n\n if (state.modifiersData[name]._skip) {\n return;\n }\n\n var _options$mainAxis = options.mainAxis,\n checkMainAxis = _options$mainAxis === void 0 ? true : _options$mainAxis,\n _options$altAxis = options.altAxis,\n checkAltAxis = _options$altAxis === void 0 ? true : _options$altAxis,\n specifiedFallbackPlacements = options.fallbackPlacements,\n padding = options.padding,\n boundary = options.boundary,\n rootBoundary = options.rootBoundary,\n altBoundary = options.altBoundary,\n _options$flipVariatio = options.flipVariations,\n flipVariations = _options$flipVariatio === void 0 ? true : _options$flipVariatio,\n allowedAutoPlacements = options.allowedAutoPlacements;\n var preferredPlacement = state.options.placement;\n var basePlacement = getBasePlacement(preferredPlacement);\n var isBasePlacement = basePlacement === preferredPlacement;\n var fallbackPlacements = specifiedFallbackPlacements || (isBasePlacement || !flipVariations ? [getOppositePlacement(preferredPlacement)] : getExpandedFallbackPlacements(preferredPlacement));\n var placements = [preferredPlacement].concat(fallbackPlacements).reduce(function (acc, placement) {\n return acc.concat(getBasePlacement(placement) === auto ? computeAutoPlacement(state, {\n placement: placement,\n boundary: boundary,\n rootBoundary: rootBoundary,\n padding: padding,\n flipVariations: flipVariations,\n allowedAutoPlacements: allowedAutoPlacements\n }) : placement);\n }, []);\n var referenceRect = state.rects.reference;\n var popperRect = state.rects.popper;\n var checksMap = new Map();\n var makeFallbackChecks = true;\n var firstFittingPlacement = placements[0];\n\n for (var i = 0; i < placements.length; i++) {\n var placement = placements[i];\n\n var _basePlacement = getBasePlacement(placement);\n\n var isStartVariation = getVariation(placement) === start;\n var isVertical = [top, bottom].indexOf(_basePlacement) >= 0;\n var len = isVertical ? 'width' : 'height';\n var overflow = detectOverflow(state, {\n placement: placement,\n boundary: boundary,\n rootBoundary: rootBoundary,\n altBoundary: altBoundary,\n padding: padding\n });\n var mainVariationSide = isVertical ? isStartVariation ? right : left : isStartVariation ? bottom : top;\n\n if (referenceRect[len] > popperRect[len]) {\n mainVariationSide = getOppositePlacement(mainVariationSide);\n }\n\n var altVariationSide = getOppositePlacement(mainVariationSide);\n var checks = [];\n\n if (checkMainAxis) {\n checks.push(overflow[_basePlacement] <= 0);\n }\n\n if (checkAltAxis) {\n checks.push(overflow[mainVariationSide] <= 0, overflow[altVariationSide] <= 0);\n }\n\n if (checks.every(function (check) {\n return check;\n })) {\n firstFittingPlacement = placement;\n makeFallbackChecks = false;\n break;\n }\n\n checksMap.set(placement, checks);\n }\n\n if (makeFallbackChecks) {\n // `2` may be desired in some cases – research later\n var numberOfChecks = flipVariations ? 3 : 1;\n\n var _loop = function _loop(_i) {\n var fittingPlacement = placements.find(function (placement) {\n var checks = checksMap.get(placement);\n\n if (checks) {\n return checks.slice(0, _i).every(function (check) {\n return check;\n });\n }\n });\n\n if (fittingPlacement) {\n firstFittingPlacement = fittingPlacement;\n return \"break\";\n }\n };\n\n for (var _i = numberOfChecks; _i > 0; _i--) {\n var _ret = _loop(_i);\n\n if (_ret === \"break\") break;\n }\n }\n\n if (state.placement !== firstFittingPlacement) {\n state.modifiersData[name]._skip = true;\n state.placement = firstFittingPlacement;\n state.reset = true;\n }\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'flip',\n enabled: true,\n phase: 'main',\n fn: flip,\n requiresIfExists: ['offset'],\n data: {\n _skip: false\n }\n};","import { top, bottom, left, right } from \"../enums.js\";\nimport detectOverflow from \"../utils/detectOverflow.js\";\n\nfunction getSideOffsets(overflow, rect, preventedOffsets) {\n if (preventedOffsets === void 0) {\n preventedOffsets = {\n x: 0,\n y: 0\n };\n }\n\n return {\n top: overflow.top - rect.height - preventedOffsets.y,\n right: overflow.right - rect.width + preventedOffsets.x,\n bottom: overflow.bottom - rect.height + preventedOffsets.y,\n left: overflow.left - rect.width - preventedOffsets.x\n };\n}\n\nfunction isAnySideFullyClipped(overflow) {\n return [top, right, bottom, left].some(function (side) {\n return overflow[side] >= 0;\n });\n}\n\nfunction hide(_ref) {\n var state = _ref.state,\n name = _ref.name;\n var referenceRect = state.rects.reference;\n var popperRect = state.rects.popper;\n var preventedOffsets = state.modifiersData.preventOverflow;\n var referenceOverflow = detectOverflow(state, {\n elementContext: 'reference'\n });\n var popperAltOverflow = detectOverflow(state, {\n altBoundary: true\n });\n var referenceClippingOffsets = getSideOffsets(referenceOverflow, referenceRect);\n var popperEscapeOffsets = getSideOffsets(popperAltOverflow, popperRect, preventedOffsets);\n var isReferenceHidden = isAnySideFullyClipped(referenceClippingOffsets);\n var hasPopperEscaped = isAnySideFullyClipped(popperEscapeOffsets);\n state.modifiersData[name] = {\n referenceClippingOffsets: referenceClippingOffsets,\n popperEscapeOffsets: popperEscapeOffsets,\n isReferenceHidden: isReferenceHidden,\n hasPopperEscaped: hasPopperEscaped\n };\n state.attributes.popper = Object.assign({}, state.attributes.popper, {\n 'data-popper-reference-hidden': isReferenceHidden,\n 'data-popper-escaped': hasPopperEscaped\n });\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'hide',\n enabled: true,\n phase: 'main',\n requiresIfExists: ['preventOverflow'],\n fn: hide\n};","import getBasePlacement from \"../utils/getBasePlacement.js\";\nimport { top, left, right, placements } from \"../enums.js\"; // eslint-disable-next-line import/no-unused-modules\n\nexport function distanceAndSkiddingToXY(placement, rects, offset) {\n var basePlacement = getBasePlacement(placement);\n var invertDistance = [left, top].indexOf(basePlacement) >= 0 ? -1 : 1;\n\n var _ref = typeof offset === 'function' ? offset(Object.assign({}, rects, {\n placement: placement\n })) : offset,\n skidding = _ref[0],\n distance = _ref[1];\n\n skidding = skidding || 0;\n distance = (distance || 0) * invertDistance;\n return [left, right].indexOf(basePlacement) >= 0 ? {\n x: distance,\n y: skidding\n } : {\n x: skidding,\n y: distance\n };\n}\n\nfunction offset(_ref2) {\n var state = _ref2.state,\n options = _ref2.options,\n name = _ref2.name;\n var _options$offset = options.offset,\n offset = _options$offset === void 0 ? [0, 0] : _options$offset;\n var data = placements.reduce(function (acc, placement) {\n acc[placement] = distanceAndSkiddingToXY(placement, state.rects, offset);\n return acc;\n }, {});\n var _data$state$placement = data[state.placement],\n x = _data$state$placement.x,\n y = _data$state$placement.y;\n\n if (state.modifiersData.popperOffsets != null) {\n state.modifiersData.popperOffsets.x += x;\n state.modifiersData.popperOffsets.y += y;\n }\n\n state.modifiersData[name] = data;\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'offset',\n enabled: true,\n phase: 'main',\n requires: ['popperOffsets'],\n fn: offset\n};","import computeOffsets from \"../utils/computeOffsets.js\";\n\nfunction popperOffsets(_ref) {\n var state = _ref.state,\n name = _ref.name;\n // Offsets are the actual position the popper needs to have to be\n // properly positioned near its reference element\n // This is the most basic placement, and will be adjusted by\n // the modifiers in the next step\n state.modifiersData[name] = computeOffsets({\n reference: state.rects.reference,\n element: state.rects.popper,\n strategy: 'absolute',\n placement: state.placement\n });\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'popperOffsets',\n enabled: true,\n phase: 'read',\n fn: popperOffsets,\n data: {}\n};","import { top, left, right, bottom, start } from \"../enums.js\";\nimport getBasePlacement from \"../utils/getBasePlacement.js\";\nimport getMainAxisFromPlacement from \"../utils/getMainAxisFromPlacement.js\";\nimport getAltAxis from \"../utils/getAltAxis.js\";\nimport { within, withinMaxClamp } from \"../utils/within.js\";\nimport getLayoutRect from \"../dom-utils/getLayoutRect.js\";\nimport getOffsetParent from \"../dom-utils/getOffsetParent.js\";\nimport detectOverflow from \"../utils/detectOverflow.js\";\nimport getVariation from \"../utils/getVariation.js\";\nimport getFreshSideObject from \"../utils/getFreshSideObject.js\";\nimport { min as mathMin, max as mathMax } from \"../utils/math.js\";\n\nfunction preventOverflow(_ref) {\n var state = _ref.state,\n options = _ref.options,\n name = _ref.name;\n var _options$mainAxis = options.mainAxis,\n checkMainAxis = _options$mainAxis === void 0 ? true : _options$mainAxis,\n _options$altAxis = options.altAxis,\n checkAltAxis = _options$altAxis === void 0 ? false : _options$altAxis,\n boundary = options.boundary,\n rootBoundary = options.rootBoundary,\n altBoundary = options.altBoundary,\n padding = options.padding,\n _options$tether = options.tether,\n tether = _options$tether === void 0 ? true : _options$tether,\n _options$tetherOffset = options.tetherOffset,\n tetherOffset = _options$tetherOffset === void 0 ? 0 : _options$tetherOffset;\n var overflow = detectOverflow(state, {\n boundary: boundary,\n rootBoundary: rootBoundary,\n padding: padding,\n altBoundary: altBoundary\n });\n var basePlacement = getBasePlacement(state.placement);\n var variation = getVariation(state.placement);\n var isBasePlacement = !variation;\n var mainAxis = getMainAxisFromPlacement(basePlacement);\n var altAxis = getAltAxis(mainAxis);\n var popperOffsets = state.modifiersData.popperOffsets;\n var referenceRect = state.rects.reference;\n var popperRect = state.rects.popper;\n var tetherOffsetValue = typeof tetherOffset === 'function' ? tetherOffset(Object.assign({}, state.rects, {\n placement: state.placement\n })) : tetherOffset;\n var normalizedTetherOffsetValue = typeof tetherOffsetValue === 'number' ? {\n mainAxis: tetherOffsetValue,\n altAxis: tetherOffsetValue\n } : Object.assign({\n mainAxis: 0,\n altAxis: 0\n }, tetherOffsetValue);\n var offsetModifierState = state.modifiersData.offset ? state.modifiersData.offset[state.placement] : null;\n var data = {\n x: 0,\n y: 0\n };\n\n if (!popperOffsets) {\n return;\n }\n\n if (checkMainAxis) {\n var _offsetModifierState$;\n\n var mainSide = mainAxis === 'y' ? top : left;\n var altSide = mainAxis === 'y' ? bottom : right;\n var len = mainAxis === 'y' ? 'height' : 'width';\n var offset = popperOffsets[mainAxis];\n var min = offset + overflow[mainSide];\n var max = offset - overflow[altSide];\n var additive = tether ? -popperRect[len] / 2 : 0;\n var minLen = variation === start ? referenceRect[len] : popperRect[len];\n var maxLen = variation === start ? -popperRect[len] : -referenceRect[len]; // We need to include the arrow in the calculation so the arrow doesn't go\n // outside the reference bounds\n\n var arrowElement = state.elements.arrow;\n var arrowRect = tether && arrowElement ? getLayoutRect(arrowElement) : {\n width: 0,\n height: 0\n };\n var arrowPaddingObject = state.modifiersData['arrow#persistent'] ? state.modifiersData['arrow#persistent'].padding : getFreshSideObject();\n var arrowPaddingMin = arrowPaddingObject[mainSide];\n var arrowPaddingMax = arrowPaddingObject[altSide]; // If the reference length is smaller than the arrow length, we don't want\n // to include its full size in the calculation. If the reference is small\n // and near the edge of a boundary, the popper can overflow even if the\n // reference is not overflowing as well (e.g. virtual elements with no\n // width or height)\n\n var arrowLen = within(0, referenceRect[len], arrowRect[len]);\n var minOffset = isBasePlacement ? referenceRect[len] / 2 - additive - arrowLen - arrowPaddingMin - normalizedTetherOffsetValue.mainAxis : minLen - arrowLen - arrowPaddingMin - normalizedTetherOffsetValue.mainAxis;\n var maxOffset = isBasePlacement ? -referenceRect[len] / 2 + additive + arrowLen + arrowPaddingMax + normalizedTetherOffsetValue.mainAxis : maxLen + arrowLen + arrowPaddingMax + normalizedTetherOffsetValue.mainAxis;\n var arrowOffsetParent = state.elements.arrow && getOffsetParent(state.elements.arrow);\n var clientOffset = arrowOffsetParent ? mainAxis === 'y' ? arrowOffsetParent.clientTop || 0 : arrowOffsetParent.clientLeft || 0 : 0;\n var offsetModifierValue = (_offsetModifierState$ = offsetModifierState == null ? void 0 : offsetModifierState[mainAxis]) != null ? _offsetModifierState$ : 0;\n var tetherMin = offset + minOffset - offsetModifierValue - clientOffset;\n var tetherMax = offset + maxOffset - offsetModifierValue;\n var preventedOffset = within(tether ? mathMin(min, tetherMin) : min, offset, tether ? mathMax(max, tetherMax) : max);\n popperOffsets[mainAxis] = preventedOffset;\n data[mainAxis] = preventedOffset - offset;\n }\n\n if (checkAltAxis) {\n var _offsetModifierState$2;\n\n var _mainSide = mainAxis === 'x' ? top : left;\n\n var _altSide = mainAxis === 'x' ? bottom : right;\n\n var _offset = popperOffsets[altAxis];\n\n var _len = altAxis === 'y' ? 'height' : 'width';\n\n var _min = _offset + overflow[_mainSide];\n\n var _max = _offset - overflow[_altSide];\n\n var isOriginSide = [top, left].indexOf(basePlacement) !== -1;\n\n var _offsetModifierValue = (_offsetModifierState$2 = offsetModifierState == null ? void 0 : offsetModifierState[altAxis]) != null ? _offsetModifierState$2 : 0;\n\n var _tetherMin = isOriginSide ? _min : _offset - referenceRect[_len] - popperRect[_len] - _offsetModifierValue + normalizedTetherOffsetValue.altAxis;\n\n var _tetherMax = isOriginSide ? _offset + referenceRect[_len] + popperRect[_len] - _offsetModifierValue - normalizedTetherOffsetValue.altAxis : _max;\n\n var _preventedOffset = tether && isOriginSide ? withinMaxClamp(_tetherMin, _offset, _tetherMax) : within(tether ? _tetherMin : _min, _offset, tether ? _tetherMax : _max);\n\n popperOffsets[altAxis] = _preventedOffset;\n data[altAxis] = _preventedOffset - _offset;\n }\n\n state.modifiersData[name] = data;\n} // eslint-disable-next-line import/no-unused-modules\n\n\nexport default {\n name: 'preventOverflow',\n enabled: true,\n phase: 'main',\n fn: preventOverflow,\n requiresIfExists: ['offset']\n};","export default function getAltAxis(axis) {\n return axis === 'x' ? 'y' : 'x';\n}","import getBoundingClientRect from \"./getBoundingClientRect.js\";\nimport getNodeScroll from \"./getNodeScroll.js\";\nimport getNodeName from \"./getNodeName.js\";\nimport { isHTMLElement } from \"./instanceOf.js\";\nimport getWindowScrollBarX from \"./getWindowScrollBarX.js\";\nimport getDocumentElement from \"./getDocumentElement.js\";\nimport isScrollParent from \"./isScrollParent.js\";\nimport { round } from \"../utils/math.js\";\n\nfunction isElementScaled(element) {\n var rect = element.getBoundingClientRect();\n var scaleX = round(rect.width) / element.offsetWidth || 1;\n var scaleY = round(rect.height) / element.offsetHeight || 1;\n return scaleX !== 1 || scaleY !== 1;\n} // Returns the composite rect of an element relative to its offsetParent.\n// Composite means it takes into account transforms as well as layout.\n\n\nexport default function getCompositeRect(elementOrVirtualElement, offsetParent, isFixed) {\n if (isFixed === void 0) {\n isFixed = false;\n }\n\n var isOffsetParentAnElement = isHTMLElement(offsetParent);\n var offsetParentIsScaled = isHTMLElement(offsetParent) && isElementScaled(offsetParent);\n var documentElement = getDocumentElement(offsetParent);\n var rect = getBoundingClientRect(elementOrVirtualElement, offsetParentIsScaled, isFixed);\n var scroll = {\n scrollLeft: 0,\n scrollTop: 0\n };\n var offsets = {\n x: 0,\n y: 0\n };\n\n if (isOffsetParentAnElement || !isOffsetParentAnElement && !isFixed) {\n if (getNodeName(offsetParent) !== 'body' || // https://github.com/popperjs/popper-core/issues/1078\n isScrollParent(documentElement)) {\n scroll = getNodeScroll(offsetParent);\n }\n\n if (isHTMLElement(offsetParent)) {\n offsets = getBoundingClientRect(offsetParent, true);\n offsets.x += offsetParent.clientLeft;\n offsets.y += offsetParent.clientTop;\n } else if (documentElement) {\n offsets.x = getWindowScrollBarX(documentElement);\n }\n }\n\n return {\n x: rect.left + scroll.scrollLeft - offsets.x,\n y: rect.top + scroll.scrollTop - offsets.y,\n width: rect.width,\n height: rect.height\n };\n}","import getWindowScroll from \"./getWindowScroll.js\";\nimport getWindow from \"./getWindow.js\";\nimport { isHTMLElement } from \"./instanceOf.js\";\nimport getHTMLElementScroll from \"./getHTMLElementScroll.js\";\nexport default function getNodeScroll(node) {\n if (node === getWindow(node) || !isHTMLElement(node)) {\n return getWindowScroll(node);\n } else {\n return getHTMLElementScroll(node);\n }\n}","export default function getHTMLElementScroll(element) {\n return {\n scrollLeft: element.scrollLeft,\n scrollTop: element.scrollTop\n };\n}","import { modifierPhases } from \"../enums.js\"; // source: https://stackoverflow.com/questions/49875255\n\nfunction order(modifiers) {\n var map = new Map();\n var visited = new Set();\n var result = [];\n modifiers.forEach(function (modifier) {\n map.set(modifier.name, modifier);\n }); // On visiting object, check for its dependencies and visit them recursively\n\n function sort(modifier) {\n visited.add(modifier.name);\n var requires = [].concat(modifier.requires || [], modifier.requiresIfExists || []);\n requires.forEach(function (dep) {\n if (!visited.has(dep)) {\n var depModifier = map.get(dep);\n\n if (depModifier) {\n sort(depModifier);\n }\n }\n });\n result.push(modifier);\n }\n\n modifiers.forEach(function (modifier) {\n if (!visited.has(modifier.name)) {\n // check for visited object\n sort(modifier);\n }\n });\n return result;\n}\n\nexport default function orderModifiers(modifiers) {\n // order based on dependencies\n var orderedModifiers = order(modifiers); // order based on phase\n\n return modifierPhases.reduce(function (acc, phase) {\n return acc.concat(orderedModifiers.filter(function (modifier) {\n return modifier.phase === phase;\n }));\n }, []);\n}","import getCompositeRect from \"./dom-utils/getCompositeRect.js\";\nimport getLayoutRect from \"./dom-utils/getLayoutRect.js\";\nimport listScrollParents from \"./dom-utils/listScrollParents.js\";\nimport getOffsetParent from \"./dom-utils/getOffsetParent.js\";\nimport orderModifiers from \"./utils/orderModifiers.js\";\nimport debounce from \"./utils/debounce.js\";\nimport mergeByName from \"./utils/mergeByName.js\";\nimport detectOverflow from \"./utils/detectOverflow.js\";\nimport { isElement } from \"./dom-utils/instanceOf.js\";\nvar DEFAULT_OPTIONS = {\n placement: 'bottom',\n modifiers: [],\n strategy: 'absolute'\n};\n\nfunction areValidElements() {\n for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {\n args[_key] = arguments[_key];\n }\n\n return !args.some(function (element) {\n return !(element && typeof element.getBoundingClientRect === 'function');\n });\n}\n\nexport function popperGenerator(generatorOptions) {\n if (generatorOptions === void 0) {\n generatorOptions = {};\n }\n\n var _generatorOptions = generatorOptions,\n _generatorOptions$def = _generatorOptions.defaultModifiers,\n defaultModifiers = _generatorOptions$def === void 0 ? [] : _generatorOptions$def,\n _generatorOptions$def2 = _generatorOptions.defaultOptions,\n defaultOptions = _generatorOptions$def2 === void 0 ? DEFAULT_OPTIONS : _generatorOptions$def2;\n return function createPopper(reference, popper, options) {\n if (options === void 0) {\n options = defaultOptions;\n }\n\n var state = {\n placement: 'bottom',\n orderedModifiers: [],\n options: Object.assign({}, DEFAULT_OPTIONS, defaultOptions),\n modifiersData: {},\n elements: {\n reference: reference,\n popper: popper\n },\n attributes: {},\n styles: {}\n };\n var effectCleanupFns = [];\n var isDestroyed = false;\n var instance = {\n state: state,\n setOptions: function setOptions(setOptionsAction) {\n var options = typeof setOptionsAction === 'function' ? setOptionsAction(state.options) : setOptionsAction;\n cleanupModifierEffects();\n state.options = Object.assign({}, defaultOptions, state.options, options);\n state.scrollParents = {\n reference: isElement(reference) ? listScrollParents(reference) : reference.contextElement ? listScrollParents(reference.contextElement) : [],\n popper: listScrollParents(popper)\n }; // Orders the modifiers based on their dependencies and `phase`\n // properties\n\n var orderedModifiers = orderModifiers(mergeByName([].concat(defaultModifiers, state.options.modifiers))); // Strip out disabled modifiers\n\n state.orderedModifiers = orderedModifiers.filter(function (m) {\n return m.enabled;\n });\n runModifierEffects();\n return instance.update();\n },\n // Sync update – it will always be executed, even if not necessary. This\n // is useful for low frequency updates where sync behavior simplifies the\n // logic.\n // For high frequency updates (e.g. `resize` and `scroll` events), always\n // prefer the async Popper#update method\n forceUpdate: function forceUpdate() {\n if (isDestroyed) {\n return;\n }\n\n var _state$elements = state.elements,\n reference = _state$elements.reference,\n popper = _state$elements.popper; // Don't proceed if `reference` or `popper` are not valid elements\n // anymore\n\n if (!areValidElements(reference, popper)) {\n return;\n } // Store the reference and popper rects to be read by modifiers\n\n\n state.rects = {\n reference: getCompositeRect(reference, getOffsetParent(popper), state.options.strategy === 'fixed'),\n popper: getLayoutRect(popper)\n }; // Modifiers have the ability to reset the current update cycle. The\n // most common use case for this is the `flip` modifier changing the\n // placement, which then needs to re-run all the modifiers, because the\n // logic was previously ran for the previous placement and is therefore\n // stale/incorrect\n\n state.reset = false;\n state.placement = state.options.placement; // On each update cycle, the `modifiersData` property for each modifier\n // is filled with the initial data specified by the modifier. This means\n // it doesn't persist and is fresh on each update.\n // To ensure persistent data, use `${name}#persistent`\n\n state.orderedModifiers.forEach(function (modifier) {\n return state.modifiersData[modifier.name] = Object.assign({}, modifier.data);\n });\n\n for (var index = 0; index < state.orderedModifiers.length; index++) {\n if (state.reset === true) {\n state.reset = false;\n index = -1;\n continue;\n }\n\n var _state$orderedModifie = state.orderedModifiers[index],\n fn = _state$orderedModifie.fn,\n _state$orderedModifie2 = _state$orderedModifie.options,\n _options = _state$orderedModifie2 === void 0 ? {} : _state$orderedModifie2,\n name = _state$orderedModifie.name;\n\n if (typeof fn === 'function') {\n state = fn({\n state: state,\n options: _options,\n name: name,\n instance: instance\n }) || state;\n }\n }\n },\n // Async and optimistically optimized update – it will not be executed if\n // not necessary (debounced to run at most once-per-tick)\n update: debounce(function () {\n return new Promise(function (resolve) {\n instance.forceUpdate();\n resolve(state);\n });\n }),\n destroy: function destroy() {\n cleanupModifierEffects();\n isDestroyed = true;\n }\n };\n\n if (!areValidElements(reference, popper)) {\n return instance;\n }\n\n instance.setOptions(options).then(function (state) {\n if (!isDestroyed && options.onFirstUpdate) {\n options.onFirstUpdate(state);\n }\n }); // Modifiers have the ability to execute arbitrary code before the first\n // update cycle runs. They will be executed in the same order as the update\n // cycle. This is useful when a modifier adds some persistent data that\n // other modifiers need to use, but the modifier is run after the dependent\n // one.\n\n function runModifierEffects() {\n state.orderedModifiers.forEach(function (_ref) {\n var name = _ref.name,\n _ref$options = _ref.options,\n options = _ref$options === void 0 ? {} : _ref$options,\n effect = _ref.effect;\n\n if (typeof effect === 'function') {\n var cleanupFn = effect({\n state: state,\n name: name,\n instance: instance,\n options: options\n });\n\n var noopFn = function noopFn() {};\n\n effectCleanupFns.push(cleanupFn || noopFn);\n }\n });\n }\n\n function cleanupModifierEffects() {\n effectCleanupFns.forEach(function (fn) {\n return fn();\n });\n effectCleanupFns = [];\n }\n\n return instance;\n };\n}\nexport var createPopper = /*#__PURE__*/popperGenerator(); // eslint-disable-next-line import/no-unused-modules\n\nexport { detectOverflow };","export default function debounce(fn) {\n var pending;\n return function () {\n if (!pending) {\n pending = new Promise(function (resolve) {\n Promise.resolve().then(function () {\n pending = undefined;\n resolve(fn());\n });\n });\n }\n\n return pending;\n };\n}","export default function mergeByName(modifiers) {\n var merged = modifiers.reduce(function (merged, current) {\n var existing = merged[current.name];\n merged[current.name] = existing ? Object.assign({}, existing, current, {\n options: Object.assign({}, existing.options, current.options),\n data: Object.assign({}, existing.data, current.data)\n }) : current;\n return merged;\n }, {}); // IE11 does not support Object.values\n\n return Object.keys(merged).map(function (key) {\n return merged[key];\n });\n}","import { popperGenerator, detectOverflow } from \"./createPopper.js\";\nimport eventListeners from \"./modifiers/eventListeners.js\";\nimport popperOffsets from \"./modifiers/popperOffsets.js\";\nimport computeStyles from \"./modifiers/computeStyles.js\";\nimport applyStyles from \"./modifiers/applyStyles.js\";\nvar defaultModifiers = [eventListeners, popperOffsets, computeStyles, applyStyles];\nvar createPopper = /*#__PURE__*/popperGenerator({\n defaultModifiers: defaultModifiers\n}); // eslint-disable-next-line import/no-unused-modules\n\nexport { createPopper, popperGenerator, defaultModifiers, detectOverflow };","import { popperGenerator, detectOverflow } from \"./createPopper.js\";\nimport eventListeners from \"./modifiers/eventListeners.js\";\nimport popperOffsets from \"./modifiers/popperOffsets.js\";\nimport computeStyles from \"./modifiers/computeStyles.js\";\nimport applyStyles from \"./modifiers/applyStyles.js\";\nimport offset from \"./modifiers/offset.js\";\nimport flip from \"./modifiers/flip.js\";\nimport preventOverflow from \"./modifiers/preventOverflow.js\";\nimport arrow from \"./modifiers/arrow.js\";\nimport hide from \"./modifiers/hide.js\";\nvar defaultModifiers = [eventListeners, popperOffsets, computeStyles, applyStyles, offset, flip, preventOverflow, arrow, hide];\nvar createPopper = /*#__PURE__*/popperGenerator({\n defaultModifiers: defaultModifiers\n}); // eslint-disable-next-line import/no-unused-modules\n\nexport { createPopper, popperGenerator, defaultModifiers, detectOverflow }; // eslint-disable-next-line import/no-unused-modules\n\nexport { createPopper as createPopperLite } from \"./popper-lite.js\"; // eslint-disable-next-line import/no-unused-modules\n\nexport * from \"./modifiers/index.js\";","/**\n * --------------------------------------------------------------------------\n * Bootstrap dropdown.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport * as Popper from '@popperjs/core'\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport Manipulator from './dom/manipulator.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport {\n defineJQueryPlugin,\n execute,\n getElement,\n getNextActiveElement,\n isDisabled,\n isElement,\n isRTL,\n isVisible,\n noop\n} from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'dropdown'\nconst DATA_KEY = 'bs.dropdown'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\n\nconst ESCAPE_KEY = 'Escape'\nconst TAB_KEY = 'Tab'\nconst ARROW_UP_KEY = 'ArrowUp'\nconst ARROW_DOWN_KEY = 'ArrowDown'\nconst RIGHT_MOUSE_BUTTON = 2 // MouseEvent.button value for the secondary button, usually the right button\n\nconst EVENT_HIDE = `hide${EVENT_KEY}`\nconst EVENT_HIDDEN = `hidden${EVENT_KEY}`\nconst EVENT_SHOW = `show${EVENT_KEY}`\nconst EVENT_SHOWN = `shown${EVENT_KEY}`\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\nconst EVENT_KEYDOWN_DATA_API = `keydown${EVENT_KEY}${DATA_API_KEY}`\nconst EVENT_KEYUP_DATA_API = `keyup${EVENT_KEY}${DATA_API_KEY}`\n\nconst CLASS_NAME_SHOW = 'show'\nconst CLASS_NAME_DROPUP = 'dropup'\nconst CLASS_NAME_DROPEND = 'dropend'\nconst CLASS_NAME_DROPSTART = 'dropstart'\nconst CLASS_NAME_DROPUP_CENTER = 'dropup-center'\nconst CLASS_NAME_DROPDOWN_CENTER = 'dropdown-center'\n\nconst SELECTOR_DATA_TOGGLE = '[data-bs-toggle=\"dropdown\"]:not(.disabled):not(:disabled)'\nconst SELECTOR_DATA_TOGGLE_SHOWN = `${SELECTOR_DATA_TOGGLE}.${CLASS_NAME_SHOW}`\nconst SELECTOR_MENU = '.dropdown-menu'\nconst SELECTOR_NAVBAR = '.navbar'\nconst SELECTOR_NAVBAR_NAV = '.navbar-nav'\nconst SELECTOR_VISIBLE_ITEMS = '.dropdown-menu .dropdown-item:not(.disabled):not(:disabled)'\n\nconst PLACEMENT_TOP = isRTL() ? 'top-end' : 'top-start'\nconst PLACEMENT_TOPEND = isRTL() ? 'top-start' : 'top-end'\nconst PLACEMENT_BOTTOM = isRTL() ? 'bottom-end' : 'bottom-start'\nconst PLACEMENT_BOTTOMEND = isRTL() ? 'bottom-start' : 'bottom-end'\nconst PLACEMENT_RIGHT = isRTL() ? 'left-start' : 'right-start'\nconst PLACEMENT_LEFT = isRTL() ? 'right-start' : 'left-start'\nconst PLACEMENT_TOPCENTER = 'top'\nconst PLACEMENT_BOTTOMCENTER = 'bottom'\n\nconst Default = {\n autoClose: true,\n boundary: 'clippingParents',\n display: 'dynamic',\n offset: [0, 2],\n popperConfig: null,\n reference: 'toggle'\n}\n\nconst DefaultType = {\n autoClose: '(boolean|string)',\n boundary: '(string|element)',\n display: 'string',\n offset: '(array|string|function)',\n popperConfig: '(null|object|function)',\n reference: '(string|element|object)'\n}\n\n/**\n * Class definition\n */\n\nclass Dropdown extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n this._popper = null\n this._parent = this._element.parentNode // dropdown wrapper\n // TODO: v6 revert #37011 & change markup https://getbootstrap.com/docs/5.3/forms/input-group/\n this._menu = SelectorEngine.next(this._element, SELECTOR_MENU)[0] ||\n SelectorEngine.prev(this._element, SELECTOR_MENU)[0] ||\n SelectorEngine.findOne(SELECTOR_MENU, this._parent)\n this._inNavbar = this._detectNavbar()\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n toggle() {\n return this._isShown() ? this.hide() : this.show()\n }\n\n show() {\n if (isDisabled(this._element) || this._isShown()) {\n return\n }\n\n const relatedTarget = {\n relatedTarget: this._element\n }\n\n const showEvent = EventHandler.trigger(this._element, EVENT_SHOW, relatedTarget)\n\n if (showEvent.defaultPrevented) {\n return\n }\n\n this._createPopper()\n\n // If this is a touch-enabled device we add extra\n // empty mouseover listeners to the body's immediate children;\n // only needed because of broken event delegation on iOS\n // https://www.quirksmode.org/blog/archives/2014/02/mouse_event_bub.html\n if ('ontouchstart' in document.documentElement && !this._parent.closest(SELECTOR_NAVBAR_NAV)) {\n for (const element of [].concat(...document.body.children)) {\n EventHandler.on(element, 'mouseover', noop)\n }\n }\n\n this._element.focus()\n this._element.setAttribute('aria-expanded', true)\n\n this._menu.classList.add(CLASS_NAME_SHOW)\n this._element.classList.add(CLASS_NAME_SHOW)\n EventHandler.trigger(this._element, EVENT_SHOWN, relatedTarget)\n }\n\n hide() {\n if (isDisabled(this._element) || !this._isShown()) {\n return\n }\n\n const relatedTarget = {\n relatedTarget: this._element\n }\n\n this._completeHide(relatedTarget)\n }\n\n dispose() {\n if (this._popper) {\n this._popper.destroy()\n }\n\n super.dispose()\n }\n\n update() {\n this._inNavbar = this._detectNavbar()\n if (this._popper) {\n this._popper.update()\n }\n }\n\n // Private\n _completeHide(relatedTarget) {\n const hideEvent = EventHandler.trigger(this._element, EVENT_HIDE, relatedTarget)\n if (hideEvent.defaultPrevented) {\n return\n }\n\n // If this is a touch-enabled device we remove the extra\n // empty mouseover listeners we added for iOS support\n if ('ontouchstart' in document.documentElement) {\n for (const element of [].concat(...document.body.children)) {\n EventHandler.off(element, 'mouseover', noop)\n }\n }\n\n if (this._popper) {\n this._popper.destroy()\n }\n\n this._menu.classList.remove(CLASS_NAME_SHOW)\n this._element.classList.remove(CLASS_NAME_SHOW)\n this._element.setAttribute('aria-expanded', 'false')\n Manipulator.removeDataAttribute(this._menu, 'popper')\n EventHandler.trigger(this._element, EVENT_HIDDEN, relatedTarget)\n }\n\n _getConfig(config) {\n config = super._getConfig(config)\n\n if (typeof config.reference === 'object' && !isElement(config.reference) &&\n typeof config.reference.getBoundingClientRect !== 'function'\n ) {\n // Popper virtual elements require a getBoundingClientRect method\n throw new TypeError(`${NAME.toUpperCase()}: Option \"reference\" provided type \"object\" without a required \"getBoundingClientRect\" method.`)\n }\n\n return config\n }\n\n _createPopper() {\n if (typeof Popper === 'undefined') {\n throw new TypeError('Bootstrap\\'s dropdowns require Popper (https://popper.js.org)')\n }\n\n let referenceElement = this._element\n\n if (this._config.reference === 'parent') {\n referenceElement = this._parent\n } else if (isElement(this._config.reference)) {\n referenceElement = getElement(this._config.reference)\n } else if (typeof this._config.reference === 'object') {\n referenceElement = this._config.reference\n }\n\n const popperConfig = this._getPopperConfig()\n this._popper = Popper.createPopper(referenceElement, this._menu, popperConfig)\n }\n\n _isShown() {\n return this._menu.classList.contains(CLASS_NAME_SHOW)\n }\n\n _getPlacement() {\n const parentDropdown = this._parent\n\n if (parentDropdown.classList.contains(CLASS_NAME_DROPEND)) {\n return PLACEMENT_RIGHT\n }\n\n if (parentDropdown.classList.contains(CLASS_NAME_DROPSTART)) {\n return PLACEMENT_LEFT\n }\n\n if (parentDropdown.classList.contains(CLASS_NAME_DROPUP_CENTER)) {\n return PLACEMENT_TOPCENTER\n }\n\n if (parentDropdown.classList.contains(CLASS_NAME_DROPDOWN_CENTER)) {\n return PLACEMENT_BOTTOMCENTER\n }\n\n // We need to trim the value because custom properties can also include spaces\n const isEnd = getComputedStyle(this._menu).getPropertyValue('--bs-position').trim() === 'end'\n\n if (parentDropdown.classList.contains(CLASS_NAME_DROPUP)) {\n return isEnd ? PLACEMENT_TOPEND : PLACEMENT_TOP\n }\n\n return isEnd ? PLACEMENT_BOTTOMEND : PLACEMENT_BOTTOM\n }\n\n _detectNavbar() {\n return this._element.closest(SELECTOR_NAVBAR) !== null\n }\n\n _getOffset() {\n const { offset } = this._config\n\n if (typeof offset === 'string') {\n return offset.split(',').map(value => Number.parseInt(value, 10))\n }\n\n if (typeof offset === 'function') {\n return popperData => offset(popperData, this._element)\n }\n\n return offset\n }\n\n _getPopperConfig() {\n const defaultBsPopperConfig = {\n placement: this._getPlacement(),\n modifiers: [{\n name: 'preventOverflow',\n options: {\n boundary: this._config.boundary\n }\n },\n {\n name: 'offset',\n options: {\n offset: this._getOffset()\n }\n }]\n }\n\n // Disable Popper if we have a static display or Dropdown is in Navbar\n if (this._inNavbar || this._config.display === 'static') {\n Manipulator.setDataAttribute(this._menu, 'popper', 'static') // TODO: v6 remove\n defaultBsPopperConfig.modifiers = [{\n name: 'applyStyles',\n enabled: false\n }]\n }\n\n return {\n ...defaultBsPopperConfig,\n ...execute(this._config.popperConfig, [defaultBsPopperConfig])\n }\n }\n\n _selectMenuItem({ key, target }) {\n const items = SelectorEngine.find(SELECTOR_VISIBLE_ITEMS, this._menu).filter(element => isVisible(element))\n\n if (!items.length) {\n return\n }\n\n // if target isn't included in items (e.g. when expanding the dropdown)\n // allow cycling to get the last item in case key equals ARROW_UP_KEY\n getNextActiveElement(items, target, key === ARROW_DOWN_KEY, !items.includes(target)).focus()\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Dropdown.getOrCreateInstance(this, config)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (typeof data[config] === 'undefined') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config]()\n })\n }\n\n static clearMenus(event) {\n if (event.button === RIGHT_MOUSE_BUTTON || (event.type === 'keyup' && event.key !== TAB_KEY)) {\n return\n }\n\n const openToggles = SelectorEngine.find(SELECTOR_DATA_TOGGLE_SHOWN)\n\n for (const toggle of openToggles) {\n const context = Dropdown.getInstance(toggle)\n if (!context || context._config.autoClose === false) {\n continue\n }\n\n const composedPath = event.composedPath()\n const isMenuTarget = composedPath.includes(context._menu)\n if (\n composedPath.includes(context._element) ||\n (context._config.autoClose === 'inside' && !isMenuTarget) ||\n (context._config.autoClose === 'outside' && isMenuTarget)\n ) {\n continue\n }\n\n // Tab navigation through the dropdown menu or events from contained inputs shouldn't close the menu\n if (context._menu.contains(event.target) && ((event.type === 'keyup' && event.key === TAB_KEY) || /input|select|option|textarea|form/i.test(event.target.tagName))) {\n continue\n }\n\n const relatedTarget = { relatedTarget: context._element }\n\n if (event.type === 'click') {\n relatedTarget.clickEvent = event\n }\n\n context._completeHide(relatedTarget)\n }\n }\n\n static dataApiKeydownHandler(event) {\n // If not an UP | DOWN | ESCAPE key => not a dropdown command\n // If input/textarea && if key is other than ESCAPE => not a dropdown command\n\n const isInput = /input|textarea/i.test(event.target.tagName)\n const isEscapeEvent = event.key === ESCAPE_KEY\n const isUpOrDownEvent = [ARROW_UP_KEY, ARROW_DOWN_KEY].includes(event.key)\n\n if (!isUpOrDownEvent && !isEscapeEvent) {\n return\n }\n\n if (isInput && !isEscapeEvent) {\n return\n }\n\n event.preventDefault()\n\n // TODO: v6 revert #37011 & change markup https://getbootstrap.com/docs/5.3/forms/input-group/\n const getToggleButton = this.matches(SELECTOR_DATA_TOGGLE) ?\n this :\n (SelectorEngine.prev(this, SELECTOR_DATA_TOGGLE)[0] ||\n SelectorEngine.next(this, SELECTOR_DATA_TOGGLE)[0] ||\n SelectorEngine.findOne(SELECTOR_DATA_TOGGLE, event.delegateTarget.parentNode))\n\n const instance = Dropdown.getOrCreateInstance(getToggleButton)\n\n if (isUpOrDownEvent) {\n event.stopPropagation()\n instance.show()\n instance._selectMenuItem(event)\n return\n }\n\n if (instance._isShown()) { // else is escape and we check if it is shown\n event.stopPropagation()\n instance.hide()\n getToggleButton.focus()\n }\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_KEYDOWN_DATA_API, SELECTOR_DATA_TOGGLE, Dropdown.dataApiKeydownHandler)\nEventHandler.on(document, EVENT_KEYDOWN_DATA_API, SELECTOR_MENU, Dropdown.dataApiKeydownHandler)\nEventHandler.on(document, EVENT_CLICK_DATA_API, Dropdown.clearMenus)\nEventHandler.on(document, EVENT_KEYUP_DATA_API, Dropdown.clearMenus)\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_TOGGLE, function (event) {\n event.preventDefault()\n Dropdown.getOrCreateInstance(this).toggle()\n})\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Dropdown)\n\nexport default Dropdown\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/backdrop.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport EventHandler from '../dom/event-handler.js'\nimport Config from './config.js'\nimport { execute, executeAfterTransition, getElement, reflow } from './index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'backdrop'\nconst CLASS_NAME_FADE = 'fade'\nconst CLASS_NAME_SHOW = 'show'\nconst EVENT_MOUSEDOWN = `mousedown.bs.${NAME}`\n\nconst Default = {\n className: 'modal-backdrop',\n clickCallback: null,\n isAnimated: false,\n isVisible: true, // if false, we use the backdrop helper without adding any element to the dom\n rootElement: 'body' // give the choice to place backdrop under different elements\n}\n\nconst DefaultType = {\n className: 'string',\n clickCallback: '(function|null)',\n isAnimated: 'boolean',\n isVisible: 'boolean',\n rootElement: '(element|string)'\n}\n\n/**\n * Class definition\n */\n\nclass Backdrop extends Config {\n constructor(config) {\n super()\n this._config = this._getConfig(config)\n this._isAppended = false\n this._element = null\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n show(callback) {\n if (!this._config.isVisible) {\n execute(callback)\n return\n }\n\n this._append()\n\n const element = this._getElement()\n if (this._config.isAnimated) {\n reflow(element)\n }\n\n element.classList.add(CLASS_NAME_SHOW)\n\n this._emulateAnimation(() => {\n execute(callback)\n })\n }\n\n hide(callback) {\n if (!this._config.isVisible) {\n execute(callback)\n return\n }\n\n this._getElement().classList.remove(CLASS_NAME_SHOW)\n\n this._emulateAnimation(() => {\n this.dispose()\n execute(callback)\n })\n }\n\n dispose() {\n if (!this._isAppended) {\n return\n }\n\n EventHandler.off(this._element, EVENT_MOUSEDOWN)\n\n this._element.remove()\n this._isAppended = false\n }\n\n // Private\n _getElement() {\n if (!this._element) {\n const backdrop = document.createElement('div')\n backdrop.className = this._config.className\n if (this._config.isAnimated) {\n backdrop.classList.add(CLASS_NAME_FADE)\n }\n\n this._element = backdrop\n }\n\n return this._element\n }\n\n _configAfterMerge(config) {\n // use getElement() with the default \"body\" to get a fresh Element on each instantiation\n config.rootElement = getElement(config.rootElement)\n return config\n }\n\n _append() {\n if (this._isAppended) {\n return\n }\n\n const element = this._getElement()\n this._config.rootElement.append(element)\n\n EventHandler.on(element, EVENT_MOUSEDOWN, () => {\n execute(this._config.clickCallback)\n })\n\n this._isAppended = true\n }\n\n _emulateAnimation(callback) {\n executeAfterTransition(callback, this._getElement(), this._config.isAnimated)\n }\n}\n\nexport default Backdrop\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/focustrap.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport EventHandler from '../dom/event-handler.js'\nimport SelectorEngine from '../dom/selector-engine.js'\nimport Config from './config.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'focustrap'\nconst DATA_KEY = 'bs.focustrap'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst EVENT_FOCUSIN = `focusin${EVENT_KEY}`\nconst EVENT_KEYDOWN_TAB = `keydown.tab${EVENT_KEY}`\n\nconst TAB_KEY = 'Tab'\nconst TAB_NAV_FORWARD = 'forward'\nconst TAB_NAV_BACKWARD = 'backward'\n\nconst Default = {\n autofocus: true,\n trapElement: null // The element to trap focus inside of\n}\n\nconst DefaultType = {\n autofocus: 'boolean',\n trapElement: 'element'\n}\n\n/**\n * Class definition\n */\n\nclass FocusTrap extends Config {\n constructor(config) {\n super()\n this._config = this._getConfig(config)\n this._isActive = false\n this._lastTabNavDirection = null\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n activate() {\n if (this._isActive) {\n return\n }\n\n if (this._config.autofocus) {\n this._config.trapElement.focus()\n }\n\n EventHandler.off(document, EVENT_KEY) // guard against infinite focus loop\n EventHandler.on(document, EVENT_FOCUSIN, event => this._handleFocusin(event))\n EventHandler.on(document, EVENT_KEYDOWN_TAB, event => this._handleKeydown(event))\n\n this._isActive = true\n }\n\n deactivate() {\n if (!this._isActive) {\n return\n }\n\n this._isActive = false\n EventHandler.off(document, EVENT_KEY)\n }\n\n // Private\n _handleFocusin(event) {\n const { trapElement } = this._config\n\n if (event.target === document || event.target === trapElement || trapElement.contains(event.target)) {\n return\n }\n\n const elements = SelectorEngine.focusableChildren(trapElement)\n\n if (elements.length === 0) {\n trapElement.focus()\n } else if (this._lastTabNavDirection === TAB_NAV_BACKWARD) {\n elements[elements.length - 1].focus()\n } else {\n elements[0].focus()\n }\n }\n\n _handleKeydown(event) {\n if (event.key !== TAB_KEY) {\n return\n }\n\n this._lastTabNavDirection = event.shiftKey ? TAB_NAV_BACKWARD : TAB_NAV_FORWARD\n }\n}\n\nexport default FocusTrap\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/scrollBar.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport Manipulator from '../dom/manipulator.js'\nimport SelectorEngine from '../dom/selector-engine.js'\nimport { isElement } from './index.js'\n\n/**\n * Constants\n */\n\nconst SELECTOR_FIXED_CONTENT = '.fixed-top, .fixed-bottom, .is-fixed, .sticky-top'\nconst SELECTOR_STICKY_CONTENT = '.sticky-top'\nconst PROPERTY_PADDING = 'padding-right'\nconst PROPERTY_MARGIN = 'margin-right'\n\n/**\n * Class definition\n */\n\nclass ScrollBarHelper {\n constructor() {\n this._element = document.body\n }\n\n // Public\n getWidth() {\n // https://developer.mozilla.org/en-US/docs/Web/API/Window/innerWidth#usage_notes\n const documentWidth = document.documentElement.clientWidth\n return Math.abs(window.innerWidth - documentWidth)\n }\n\n hide() {\n const width = this.getWidth()\n this._disableOverFlow()\n // give padding to element to balance the hidden scrollbar width\n this._setElementAttributes(this._element, PROPERTY_PADDING, calculatedValue => calculatedValue + width)\n // trick: We adjust positive paddingRight and negative marginRight to sticky-top elements to keep showing fullwidth\n this._setElementAttributes(SELECTOR_FIXED_CONTENT, PROPERTY_PADDING, calculatedValue => calculatedValue + width)\n this._setElementAttributes(SELECTOR_STICKY_CONTENT, PROPERTY_MARGIN, calculatedValue => calculatedValue - width)\n }\n\n reset() {\n this._resetElementAttributes(this._element, 'overflow')\n this._resetElementAttributes(this._element, PROPERTY_PADDING)\n this._resetElementAttributes(SELECTOR_FIXED_CONTENT, PROPERTY_PADDING)\n this._resetElementAttributes(SELECTOR_STICKY_CONTENT, PROPERTY_MARGIN)\n }\n\n isOverflowing() {\n return this.getWidth() > 0\n }\n\n // Private\n _disableOverFlow() {\n this._saveInitialAttribute(this._element, 'overflow')\n this._element.style.overflow = 'hidden'\n }\n\n _setElementAttributes(selector, styleProperty, callback) {\n const scrollbarWidth = this.getWidth()\n const manipulationCallBack = element => {\n if (element !== this._element && window.innerWidth > element.clientWidth + scrollbarWidth) {\n return\n }\n\n this._saveInitialAttribute(element, styleProperty)\n const calculatedValue = window.getComputedStyle(element).getPropertyValue(styleProperty)\n element.style.setProperty(styleProperty, `${callback(Number.parseFloat(calculatedValue))}px`)\n }\n\n this._applyManipulationCallback(selector, manipulationCallBack)\n }\n\n _saveInitialAttribute(element, styleProperty) {\n const actualValue = element.style.getPropertyValue(styleProperty)\n if (actualValue) {\n Manipulator.setDataAttribute(element, styleProperty, actualValue)\n }\n }\n\n _resetElementAttributes(selector, styleProperty) {\n const manipulationCallBack = element => {\n const value = Manipulator.getDataAttribute(element, styleProperty)\n // We only want to remove the property if the value is `null`; the value can also be zero\n if (value === null) {\n element.style.removeProperty(styleProperty)\n return\n }\n\n Manipulator.removeDataAttribute(element, styleProperty)\n element.style.setProperty(styleProperty, value)\n }\n\n this._applyManipulationCallback(selector, manipulationCallBack)\n }\n\n _applyManipulationCallback(selector, callBack) {\n if (isElement(selector)) {\n callBack(selector)\n return\n }\n\n for (const sel of SelectorEngine.find(selector, this._element)) {\n callBack(sel)\n }\n }\n}\n\nexport default ScrollBarHelper\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap modal.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport Backdrop from './util/backdrop.js'\nimport { enableDismissTrigger } from './util/component-functions.js'\nimport FocusTrap from './util/focustrap.js'\nimport { defineJQueryPlugin, isRTL, isVisible, reflow } from './util/index.js'\nimport ScrollBarHelper from './util/scrollbar.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'modal'\nconst DATA_KEY = 'bs.modal'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\nconst ESCAPE_KEY = 'Escape'\n\nconst EVENT_HIDE = `hide${EVENT_KEY}`\nconst EVENT_HIDE_PREVENTED = `hidePrevented${EVENT_KEY}`\nconst EVENT_HIDDEN = `hidden${EVENT_KEY}`\nconst EVENT_SHOW = `show${EVENT_KEY}`\nconst EVENT_SHOWN = `shown${EVENT_KEY}`\nconst EVENT_RESIZE = `resize${EVENT_KEY}`\nconst EVENT_CLICK_DISMISS = `click.dismiss${EVENT_KEY}`\nconst EVENT_MOUSEDOWN_DISMISS = `mousedown.dismiss${EVENT_KEY}`\nconst EVENT_KEYDOWN_DISMISS = `keydown.dismiss${EVENT_KEY}`\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\n\nconst CLASS_NAME_OPEN = 'modal-open'\nconst CLASS_NAME_FADE = 'fade'\nconst CLASS_NAME_SHOW = 'show'\nconst CLASS_NAME_STATIC = 'modal-static'\n\nconst OPEN_SELECTOR = '.modal.show'\nconst SELECTOR_DIALOG = '.modal-dialog'\nconst SELECTOR_MODAL_BODY = '.modal-body'\nconst SELECTOR_DATA_TOGGLE = '[data-bs-toggle=\"modal\"]'\n\nconst Default = {\n backdrop: true,\n focus: true,\n keyboard: true\n}\n\nconst DefaultType = {\n backdrop: '(boolean|string)',\n focus: 'boolean',\n keyboard: 'boolean'\n}\n\n/**\n * Class definition\n */\n\nclass Modal extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n this._dialog = SelectorEngine.findOne(SELECTOR_DIALOG, this._element)\n this._backdrop = this._initializeBackDrop()\n this._focustrap = this._initializeFocusTrap()\n this._isShown = false\n this._isTransitioning = false\n this._scrollBar = new ScrollBarHelper()\n\n this._addEventListeners()\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n toggle(relatedTarget) {\n return this._isShown ? this.hide() : this.show(relatedTarget)\n }\n\n show(relatedTarget) {\n if (this._isShown || this._isTransitioning) {\n return\n }\n\n const showEvent = EventHandler.trigger(this._element, EVENT_SHOW, {\n relatedTarget\n })\n\n if (showEvent.defaultPrevented) {\n return\n }\n\n this._isShown = true\n this._isTransitioning = true\n\n this._scrollBar.hide()\n\n document.body.classList.add(CLASS_NAME_OPEN)\n\n this._adjustDialog()\n\n this._backdrop.show(() => this._showElement(relatedTarget))\n }\n\n hide() {\n if (!this._isShown || this._isTransitioning) {\n return\n }\n\n const hideEvent = EventHandler.trigger(this._element, EVENT_HIDE)\n\n if (hideEvent.defaultPrevented) {\n return\n }\n\n this._isShown = false\n this._isTransitioning = true\n this._focustrap.deactivate()\n\n this._element.classList.remove(CLASS_NAME_SHOW)\n\n this._queueCallback(() => this._hideModal(), this._element, this._isAnimated())\n }\n\n dispose() {\n EventHandler.off(window, EVENT_KEY)\n EventHandler.off(this._dialog, EVENT_KEY)\n\n this._backdrop.dispose()\n this._focustrap.deactivate()\n\n super.dispose()\n }\n\n handleUpdate() {\n this._adjustDialog()\n }\n\n // Private\n _initializeBackDrop() {\n return new Backdrop({\n isVisible: Boolean(this._config.backdrop), // 'static' option will be translated to true, and booleans will keep their value,\n isAnimated: this._isAnimated()\n })\n }\n\n _initializeFocusTrap() {\n return new FocusTrap({\n trapElement: this._element\n })\n }\n\n _showElement(relatedTarget) {\n // try to append dynamic modal\n if (!document.body.contains(this._element)) {\n document.body.append(this._element)\n }\n\n this._element.style.display = 'block'\n this._element.removeAttribute('aria-hidden')\n this._element.setAttribute('aria-modal', true)\n this._element.setAttribute('role', 'dialog')\n this._element.scrollTop = 0\n\n const modalBody = SelectorEngine.findOne(SELECTOR_MODAL_BODY, this._dialog)\n if (modalBody) {\n modalBody.scrollTop = 0\n }\n\n reflow(this._element)\n\n this._element.classList.add(CLASS_NAME_SHOW)\n\n const transitionComplete = () => {\n if (this._config.focus) {\n this._focustrap.activate()\n }\n\n this._isTransitioning = false\n EventHandler.trigger(this._element, EVENT_SHOWN, {\n relatedTarget\n })\n }\n\n this._queueCallback(transitionComplete, this._dialog, this._isAnimated())\n }\n\n _addEventListeners() {\n EventHandler.on(this._element, EVENT_KEYDOWN_DISMISS, event => {\n if (event.key !== ESCAPE_KEY) {\n return\n }\n\n if (this._config.keyboard) {\n this.hide()\n return\n }\n\n this._triggerBackdropTransition()\n })\n\n EventHandler.on(window, EVENT_RESIZE, () => {\n if (this._isShown && !this._isTransitioning) {\n this._adjustDialog()\n }\n })\n\n EventHandler.on(this._element, EVENT_MOUSEDOWN_DISMISS, event => {\n // a bad trick to segregate clicks that may start inside dialog but end outside, and avoid listen to scrollbar clicks\n EventHandler.one(this._element, EVENT_CLICK_DISMISS, event2 => {\n if (this._element !== event.target || this._element !== event2.target) {\n return\n }\n\n if (this._config.backdrop === 'static') {\n this._triggerBackdropTransition()\n return\n }\n\n if (this._config.backdrop) {\n this.hide()\n }\n })\n })\n }\n\n _hideModal() {\n this._element.style.display = 'none'\n this._element.setAttribute('aria-hidden', true)\n this._element.removeAttribute('aria-modal')\n this._element.removeAttribute('role')\n this._isTransitioning = false\n\n this._backdrop.hide(() => {\n document.body.classList.remove(CLASS_NAME_OPEN)\n this._resetAdjustments()\n this._scrollBar.reset()\n EventHandler.trigger(this._element, EVENT_HIDDEN)\n })\n }\n\n _isAnimated() {\n return this._element.classList.contains(CLASS_NAME_FADE)\n }\n\n _triggerBackdropTransition() {\n const hideEvent = EventHandler.trigger(this._element, EVENT_HIDE_PREVENTED)\n if (hideEvent.defaultPrevented) {\n return\n }\n\n const isModalOverflowing = this._element.scrollHeight > document.documentElement.clientHeight\n const initialOverflowY = this._element.style.overflowY\n // return if the following background transition hasn't yet completed\n if (initialOverflowY === 'hidden' || this._element.classList.contains(CLASS_NAME_STATIC)) {\n return\n }\n\n if (!isModalOverflowing) {\n this._element.style.overflowY = 'hidden'\n }\n\n this._element.classList.add(CLASS_NAME_STATIC)\n this._queueCallback(() => {\n this._element.classList.remove(CLASS_NAME_STATIC)\n this._queueCallback(() => {\n this._element.style.overflowY = initialOverflowY\n }, this._dialog)\n }, this._dialog)\n\n this._element.focus()\n }\n\n /**\n * The following methods are used to handle overflowing modals\n */\n\n _adjustDialog() {\n const isModalOverflowing = this._element.scrollHeight > document.documentElement.clientHeight\n const scrollbarWidth = this._scrollBar.getWidth()\n const isBodyOverflowing = scrollbarWidth > 0\n\n if (isBodyOverflowing && !isModalOverflowing) {\n const property = isRTL() ? 'paddingLeft' : 'paddingRight'\n this._element.style[property] = `${scrollbarWidth}px`\n }\n\n if (!isBodyOverflowing && isModalOverflowing) {\n const property = isRTL() ? 'paddingRight' : 'paddingLeft'\n this._element.style[property] = `${scrollbarWidth}px`\n }\n }\n\n _resetAdjustments() {\n this._element.style.paddingLeft = ''\n this._element.style.paddingRight = ''\n }\n\n // Static\n static jQueryInterface(config, relatedTarget) {\n return this.each(function () {\n const data = Modal.getOrCreateInstance(this, config)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (typeof data[config] === 'undefined') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config](relatedTarget)\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_TOGGLE, function (event) {\n const target = SelectorEngine.getElementFromSelector(this)\n\n if (['A', 'AREA'].includes(this.tagName)) {\n event.preventDefault()\n }\n\n EventHandler.one(target, EVENT_SHOW, showEvent => {\n if (showEvent.defaultPrevented) {\n // only register focus restorer if modal will actually get shown\n return\n }\n\n EventHandler.one(target, EVENT_HIDDEN, () => {\n if (isVisible(this)) {\n this.focus()\n }\n })\n })\n\n // avoid conflict when clicking modal toggler while another one is open\n const alreadyOpen = SelectorEngine.findOne(OPEN_SELECTOR)\n if (alreadyOpen) {\n Modal.getInstance(alreadyOpen).hide()\n }\n\n const data = Modal.getOrCreateInstance(target)\n\n data.toggle(this)\n})\n\nenableDismissTrigger(Modal)\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Modal)\n\nexport default Modal\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap offcanvas.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport Backdrop from './util/backdrop.js'\nimport { enableDismissTrigger } from './util/component-functions.js'\nimport FocusTrap from './util/focustrap.js'\nimport {\n defineJQueryPlugin,\n isDisabled,\n isVisible\n} from './util/index.js'\nimport ScrollBarHelper from './util/scrollbar.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'offcanvas'\nconst DATA_KEY = 'bs.offcanvas'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\nconst EVENT_LOAD_DATA_API = `load${EVENT_KEY}${DATA_API_KEY}`\nconst ESCAPE_KEY = 'Escape'\n\nconst CLASS_NAME_SHOW = 'show'\nconst CLASS_NAME_SHOWING = 'showing'\nconst CLASS_NAME_HIDING = 'hiding'\nconst CLASS_NAME_BACKDROP = 'offcanvas-backdrop'\nconst OPEN_SELECTOR = '.offcanvas.show'\n\nconst EVENT_SHOW = `show${EVENT_KEY}`\nconst EVENT_SHOWN = `shown${EVENT_KEY}`\nconst EVENT_HIDE = `hide${EVENT_KEY}`\nconst EVENT_HIDE_PREVENTED = `hidePrevented${EVENT_KEY}`\nconst EVENT_HIDDEN = `hidden${EVENT_KEY}`\nconst EVENT_RESIZE = `resize${EVENT_KEY}`\nconst EVENT_CLICK_DATA_API = `click${EVENT_KEY}${DATA_API_KEY}`\nconst EVENT_KEYDOWN_DISMISS = `keydown.dismiss${EVENT_KEY}`\n\nconst SELECTOR_DATA_TOGGLE = '[data-bs-toggle=\"offcanvas\"]'\n\nconst Default = {\n backdrop: true,\n keyboard: true,\n scroll: false\n}\n\nconst DefaultType = {\n backdrop: '(boolean|string)',\n keyboard: 'boolean',\n scroll: 'boolean'\n}\n\n/**\n * Class definition\n */\n\nclass Offcanvas extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n this._isShown = false\n this._backdrop = this._initializeBackDrop()\n this._focustrap = this._initializeFocusTrap()\n this._addEventListeners()\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n toggle(relatedTarget) {\n return this._isShown ? this.hide() : this.show(relatedTarget)\n }\n\n show(relatedTarget) {\n if (this._isShown) {\n return\n }\n\n const showEvent = EventHandler.trigger(this._element, EVENT_SHOW, { relatedTarget })\n\n if (showEvent.defaultPrevented) {\n return\n }\n\n this._isShown = true\n this._backdrop.show()\n\n if (!this._config.scroll) {\n new ScrollBarHelper().hide()\n }\n\n this._element.setAttribute('aria-modal', true)\n this._element.setAttribute('role', 'dialog')\n this._element.classList.add(CLASS_NAME_SHOWING)\n\n const completeCallBack = () => {\n if (!this._config.scroll || this._config.backdrop) {\n this._focustrap.activate()\n }\n\n this._element.classList.add(CLASS_NAME_SHOW)\n this._element.classList.remove(CLASS_NAME_SHOWING)\n EventHandler.trigger(this._element, EVENT_SHOWN, { relatedTarget })\n }\n\n this._queueCallback(completeCallBack, this._element, true)\n }\n\n hide() {\n if (!this._isShown) {\n return\n }\n\n const hideEvent = EventHandler.trigger(this._element, EVENT_HIDE)\n\n if (hideEvent.defaultPrevented) {\n return\n }\n\n this._focustrap.deactivate()\n this._element.blur()\n this._isShown = false\n this._element.classList.add(CLASS_NAME_HIDING)\n this._backdrop.hide()\n\n const completeCallback = () => {\n this._element.classList.remove(CLASS_NAME_SHOW, CLASS_NAME_HIDING)\n this._element.removeAttribute('aria-modal')\n this._element.removeAttribute('role')\n\n if (!this._config.scroll) {\n new ScrollBarHelper().reset()\n }\n\n EventHandler.trigger(this._element, EVENT_HIDDEN)\n }\n\n this._queueCallback(completeCallback, this._element, true)\n }\n\n dispose() {\n this._backdrop.dispose()\n this._focustrap.deactivate()\n super.dispose()\n }\n\n // Private\n _initializeBackDrop() {\n const clickCallback = () => {\n if (this._config.backdrop === 'static') {\n EventHandler.trigger(this._element, EVENT_HIDE_PREVENTED)\n return\n }\n\n this.hide()\n }\n\n // 'static' option will be translated to true, and booleans will keep their value\n const isVisible = Boolean(this._config.backdrop)\n\n return new Backdrop({\n className: CLASS_NAME_BACKDROP,\n isVisible,\n isAnimated: true,\n rootElement: this._element.parentNode,\n clickCallback: isVisible ? clickCallback : null\n })\n }\n\n _initializeFocusTrap() {\n return new FocusTrap({\n trapElement: this._element\n })\n }\n\n _addEventListeners() {\n EventHandler.on(this._element, EVENT_KEYDOWN_DISMISS, event => {\n if (event.key !== ESCAPE_KEY) {\n return\n }\n\n if (this._config.keyboard) {\n this.hide()\n return\n }\n\n EventHandler.trigger(this._element, EVENT_HIDE_PREVENTED)\n })\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Offcanvas.getOrCreateInstance(this, config)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (data[config] === undefined || config.startsWith('_') || config === 'constructor') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config](this)\n })\n }\n}\n\n/**\n * Data API implementation\n */\n\nEventHandler.on(document, EVENT_CLICK_DATA_API, SELECTOR_DATA_TOGGLE, function (event) {\n const target = SelectorEngine.getElementFromSelector(this)\n\n if (['A', 'AREA'].includes(this.tagName)) {\n event.preventDefault()\n }\n\n if (isDisabled(this)) {\n return\n }\n\n EventHandler.one(target, EVENT_HIDDEN, () => {\n // focus on trigger when it is closed\n if (isVisible(this)) {\n this.focus()\n }\n })\n\n // avoid conflict when clicking a toggler of an offcanvas, while another is open\n const alreadyOpen = SelectorEngine.findOne(OPEN_SELECTOR)\n if (alreadyOpen && alreadyOpen !== target) {\n Offcanvas.getInstance(alreadyOpen).hide()\n }\n\n const data = Offcanvas.getOrCreateInstance(target)\n data.toggle(this)\n})\n\nEventHandler.on(window, EVENT_LOAD_DATA_API, () => {\n for (const selector of SelectorEngine.find(OPEN_SELECTOR)) {\n Offcanvas.getOrCreateInstance(selector).show()\n }\n})\n\nEventHandler.on(window, EVENT_RESIZE, () => {\n for (const element of SelectorEngine.find('[aria-modal][class*=show][class*=offcanvas-]')) {\n if (getComputedStyle(element).position !== 'fixed') {\n Offcanvas.getOrCreateInstance(element).hide()\n }\n }\n})\n\nenableDismissTrigger(Offcanvas)\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Offcanvas)\n\nexport default Offcanvas\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/sanitizer.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\n// js-docs-start allow-list\nconst ARIA_ATTRIBUTE_PATTERN = /^aria-[\\w-]*$/i\n\nexport const DefaultAllowlist = {\n // Global attributes allowed on any supplied element below.\n '*': ['class', 'dir', 'id', 'lang', 'role', ARIA_ATTRIBUTE_PATTERN],\n a: ['target', 'href', 'title', 'rel'],\n area: [],\n b: [],\n br: [],\n col: [],\n code: [],\n div: [],\n em: [],\n hr: [],\n h1: [],\n h2: [],\n h3: [],\n h4: [],\n h5: [],\n h6: [],\n i: [],\n img: ['src', 'srcset', 'alt', 'title', 'width', 'height'],\n li: [],\n ol: [],\n p: [],\n pre: [],\n s: [],\n small: [],\n span: [],\n sub: [],\n sup: [],\n strong: [],\n u: [],\n ul: []\n}\n// js-docs-end allow-list\n\nconst uriAttributes = new Set([\n 'background',\n 'cite',\n 'href',\n 'itemtype',\n 'longdesc',\n 'poster',\n 'src',\n 'xlink:href'\n])\n\n/**\n * A pattern that recognizes URLs that are safe wrt. XSS in URL navigation\n * contexts.\n *\n * Shout-out to Angular https://github.com/angular/angular/blob/15.2.8/packages/core/src/sanitization/url_sanitizer.ts#L38\n */\n// eslint-disable-next-line unicorn/better-regex\nconst SAFE_URL_PATTERN = /^(?!javascript:)(?:[a-z0-9+.-]+:|[^&:/?#]*(?:[/?#]|$))/i\n\nconst allowedAttribute = (attribute, allowedAttributeList) => {\n const attributeName = attribute.nodeName.toLowerCase()\n\n if (allowedAttributeList.includes(attributeName)) {\n if (uriAttributes.has(attributeName)) {\n return Boolean(SAFE_URL_PATTERN.test(attribute.nodeValue))\n }\n\n return true\n }\n\n // Check if a regular expression validates the attribute.\n return allowedAttributeList.filter(attributeRegex => attributeRegex instanceof RegExp)\n .some(regex => regex.test(attributeName))\n}\n\nexport function sanitizeHtml(unsafeHtml, allowList, sanitizeFunction) {\n if (!unsafeHtml.length) {\n return unsafeHtml\n }\n\n if (sanitizeFunction && typeof sanitizeFunction === 'function') {\n return sanitizeFunction(unsafeHtml)\n }\n\n const domParser = new window.DOMParser()\n const createdDocument = domParser.parseFromString(unsafeHtml, 'text/html')\n const elements = [].concat(...createdDocument.body.querySelectorAll('*'))\n\n for (const element of elements) {\n const elementName = element.nodeName.toLowerCase()\n\n if (!Object.keys(allowList).includes(elementName)) {\n element.remove()\n continue\n }\n\n const attributeList = [].concat(...element.attributes)\n const allowedAttributes = [].concat(allowList['*'] || [], allowList[elementName] || [])\n\n for (const attribute of attributeList) {\n if (!allowedAttribute(attribute, allowedAttributes)) {\n element.removeAttribute(attribute.nodeName)\n }\n }\n }\n\n return createdDocument.body.innerHTML\n}\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap util/template-factory.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport SelectorEngine from '../dom/selector-engine.js'\nimport Config from './config.js'\nimport { DefaultAllowlist, sanitizeHtml } from './sanitizer.js'\nimport { execute, getElement, isElement } from './index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'TemplateFactory'\n\nconst Default = {\n allowList: DefaultAllowlist,\n content: {}, // { selector : text , selector2 : text2 , }\n extraClass: '',\n html: false,\n sanitize: true,\n sanitizeFn: null,\n template: '
'\n}\n\nconst DefaultType = {\n allowList: 'object',\n content: 'object',\n extraClass: '(string|function)',\n html: 'boolean',\n sanitize: 'boolean',\n sanitizeFn: '(null|function)',\n template: 'string'\n}\n\nconst DefaultContentType = {\n entry: '(string|element|function|null)',\n selector: '(string|element)'\n}\n\n/**\n * Class definition\n */\n\nclass TemplateFactory extends Config {\n constructor(config) {\n super()\n this._config = this._getConfig(config)\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n getContent() {\n return Object.values(this._config.content)\n .map(config => this._resolvePossibleFunction(config))\n .filter(Boolean)\n }\n\n hasContent() {\n return this.getContent().length > 0\n }\n\n changeContent(content) {\n this._checkContent(content)\n this._config.content = { ...this._config.content, ...content }\n return this\n }\n\n toHtml() {\n const templateWrapper = document.createElement('div')\n templateWrapper.innerHTML = this._maybeSanitize(this._config.template)\n\n for (const [selector, text] of Object.entries(this._config.content)) {\n this._setContent(templateWrapper, text, selector)\n }\n\n const template = templateWrapper.children[0]\n const extraClass = this._resolvePossibleFunction(this._config.extraClass)\n\n if (extraClass) {\n template.classList.add(...extraClass.split(' '))\n }\n\n return template\n }\n\n // Private\n _typeCheckConfig(config) {\n super._typeCheckConfig(config)\n this._checkContent(config.content)\n }\n\n _checkContent(arg) {\n for (const [selector, content] of Object.entries(arg)) {\n super._typeCheckConfig({ selector, entry: content }, DefaultContentType)\n }\n }\n\n _setContent(template, content, selector) {\n const templateElement = SelectorEngine.findOne(selector, template)\n\n if (!templateElement) {\n return\n }\n\n content = this._resolvePossibleFunction(content)\n\n if (!content) {\n templateElement.remove()\n return\n }\n\n if (isElement(content)) {\n this._putElementInTemplate(getElement(content), templateElement)\n return\n }\n\n if (this._config.html) {\n templateElement.innerHTML = this._maybeSanitize(content)\n return\n }\n\n templateElement.textContent = content\n }\n\n _maybeSanitize(arg) {\n return this._config.sanitize ? sanitizeHtml(arg, this._config.allowList, this._config.sanitizeFn) : arg\n }\n\n _resolvePossibleFunction(arg) {\n return execute(arg, [this])\n }\n\n _putElementInTemplate(element, templateElement) {\n if (this._config.html) {\n templateElement.innerHTML = ''\n templateElement.append(element)\n return\n }\n\n templateElement.textContent = element.textContent\n }\n}\n\nexport default TemplateFactory\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap tooltip.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport * as Popper from '@popperjs/core'\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport Manipulator from './dom/manipulator.js'\nimport { defineJQueryPlugin, execute, findShadowRoot, getElement, getUID, isRTL, noop } from './util/index.js'\nimport { DefaultAllowlist } from './util/sanitizer.js'\nimport TemplateFactory from './util/template-factory.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'tooltip'\nconst DISALLOWED_ATTRIBUTES = new Set(['sanitize', 'allowList', 'sanitizeFn'])\n\nconst CLASS_NAME_FADE = 'fade'\nconst CLASS_NAME_MODAL = 'modal'\nconst CLASS_NAME_SHOW = 'show'\n\nconst SELECTOR_TOOLTIP_INNER = '.tooltip-inner'\nconst SELECTOR_MODAL = `.${CLASS_NAME_MODAL}`\n\nconst EVENT_MODAL_HIDE = 'hide.bs.modal'\n\nconst TRIGGER_HOVER = 'hover'\nconst TRIGGER_FOCUS = 'focus'\nconst TRIGGER_CLICK = 'click'\nconst TRIGGER_MANUAL = 'manual'\n\nconst EVENT_HIDE = 'hide'\nconst EVENT_HIDDEN = 'hidden'\nconst EVENT_SHOW = 'show'\nconst EVENT_SHOWN = 'shown'\nconst EVENT_INSERTED = 'inserted'\nconst EVENT_CLICK = 'click'\nconst EVENT_FOCUSIN = 'focusin'\nconst EVENT_FOCUSOUT = 'focusout'\nconst EVENT_MOUSEENTER = 'mouseenter'\nconst EVENT_MOUSELEAVE = 'mouseleave'\n\nconst AttachmentMap = {\n AUTO: 'auto',\n TOP: 'top',\n RIGHT: isRTL() ? 'left' : 'right',\n BOTTOM: 'bottom',\n LEFT: isRTL() ? 'right' : 'left'\n}\n\nconst Default = {\n allowList: DefaultAllowlist,\n animation: true,\n boundary: 'clippingParents',\n container: false,\n customClass: '',\n delay: 0,\n fallbackPlacements: ['top', 'right', 'bottom', 'left'],\n html: false,\n offset: [0, 6],\n placement: 'top',\n popperConfig: null,\n sanitize: true,\n sanitizeFn: null,\n selector: false,\n template: '
' +\n '
' +\n '
' +\n '
',\n title: '',\n trigger: 'hover focus'\n}\n\nconst DefaultType = {\n allowList: 'object',\n animation: 'boolean',\n boundary: '(string|element)',\n container: '(string|element|boolean)',\n customClass: '(string|function)',\n delay: '(number|object)',\n fallbackPlacements: 'array',\n html: 'boolean',\n offset: '(array|string|function)',\n placement: '(string|function)',\n popperConfig: '(null|object|function)',\n sanitize: 'boolean',\n sanitizeFn: '(null|function)',\n selector: '(string|boolean)',\n template: 'string',\n title: '(string|element|function)',\n trigger: 'string'\n}\n\n/**\n * Class definition\n */\n\nclass Tooltip extends BaseComponent {\n constructor(element, config) {\n if (typeof Popper === 'undefined') {\n throw new TypeError('Bootstrap\\'s tooltips require Popper (https://popper.js.org)')\n }\n\n super(element, config)\n\n // Private\n this._isEnabled = true\n this._timeout = 0\n this._isHovered = null\n this._activeTrigger = {}\n this._popper = null\n this._templateFactory = null\n this._newContent = null\n\n // Protected\n this.tip = null\n\n this._setListeners()\n\n if (!this._config.selector) {\n this._fixTitle()\n }\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n enable() {\n this._isEnabled = true\n }\n\n disable() {\n this._isEnabled = false\n }\n\n toggleEnabled() {\n this._isEnabled = !this._isEnabled\n }\n\n toggle() {\n if (!this._isEnabled) {\n return\n }\n\n this._activeTrigger.click = !this._activeTrigger.click\n if (this._isShown()) {\n this._leave()\n return\n }\n\n this._enter()\n }\n\n dispose() {\n clearTimeout(this._timeout)\n\n EventHandler.off(this._element.closest(SELECTOR_MODAL), EVENT_MODAL_HIDE, this._hideModalHandler)\n\n if (this._element.getAttribute('data-bs-original-title')) {\n this._element.setAttribute('title', this._element.getAttribute('data-bs-original-title'))\n }\n\n this._disposePopper()\n super.dispose()\n }\n\n show() {\n if (this._element.style.display === 'none') {\n throw new Error('Please use show on visible elements')\n }\n\n if (!(this._isWithContent() && this._isEnabled)) {\n return\n }\n\n const showEvent = EventHandler.trigger(this._element, this.constructor.eventName(EVENT_SHOW))\n const shadowRoot = findShadowRoot(this._element)\n const isInTheDom = (shadowRoot || this._element.ownerDocument.documentElement).contains(this._element)\n\n if (showEvent.defaultPrevented || !isInTheDom) {\n return\n }\n\n // TODO: v6 remove this or make it optional\n this._disposePopper()\n\n const tip = this._getTipElement()\n\n this._element.setAttribute('aria-describedby', tip.getAttribute('id'))\n\n const { container } = this._config\n\n if (!this._element.ownerDocument.documentElement.contains(this.tip)) {\n container.append(tip)\n EventHandler.trigger(this._element, this.constructor.eventName(EVENT_INSERTED))\n }\n\n this._popper = this._createPopper(tip)\n\n tip.classList.add(CLASS_NAME_SHOW)\n\n // If this is a touch-enabled device we add extra\n // empty mouseover listeners to the body's immediate children;\n // only needed because of broken event delegation on iOS\n // https://www.quirksmode.org/blog/archives/2014/02/mouse_event_bub.html\n if ('ontouchstart' in document.documentElement) {\n for (const element of [].concat(...document.body.children)) {\n EventHandler.on(element, 'mouseover', noop)\n }\n }\n\n const complete = () => {\n EventHandler.trigger(this._element, this.constructor.eventName(EVENT_SHOWN))\n\n if (this._isHovered === false) {\n this._leave()\n }\n\n this._isHovered = false\n }\n\n this._queueCallback(complete, this.tip, this._isAnimated())\n }\n\n hide() {\n if (!this._isShown()) {\n return\n }\n\n const hideEvent = EventHandler.trigger(this._element, this.constructor.eventName(EVENT_HIDE))\n if (hideEvent.defaultPrevented) {\n return\n }\n\n const tip = this._getTipElement()\n tip.classList.remove(CLASS_NAME_SHOW)\n\n // If this is a touch-enabled device we remove the extra\n // empty mouseover listeners we added for iOS support\n if ('ontouchstart' in document.documentElement) {\n for (const element of [].concat(...document.body.children)) {\n EventHandler.off(element, 'mouseover', noop)\n }\n }\n\n this._activeTrigger[TRIGGER_CLICK] = false\n this._activeTrigger[TRIGGER_FOCUS] = false\n this._activeTrigger[TRIGGER_HOVER] = false\n this._isHovered = null // it is a trick to support manual triggering\n\n const complete = () => {\n if (this._isWithActiveTrigger()) {\n return\n }\n\n if (!this._isHovered) {\n this._disposePopper()\n }\n\n this._element.removeAttribute('aria-describedby')\n EventHandler.trigger(this._element, this.constructor.eventName(EVENT_HIDDEN))\n }\n\n this._queueCallback(complete, this.tip, this._isAnimated())\n }\n\n update() {\n if (this._popper) {\n this._popper.update()\n }\n }\n\n // Protected\n _isWithContent() {\n return Boolean(this._getTitle())\n }\n\n _getTipElement() {\n if (!this.tip) {\n this.tip = this._createTipElement(this._newContent || this._getContentForTemplate())\n }\n\n return this.tip\n }\n\n _createTipElement(content) {\n const tip = this._getTemplateFactory(content).toHtml()\n\n // TODO: remove this check in v6\n if (!tip) {\n return null\n }\n\n tip.classList.remove(CLASS_NAME_FADE, CLASS_NAME_SHOW)\n // TODO: v6 the following can be achieved with CSS only\n tip.classList.add(`bs-${this.constructor.NAME}-auto`)\n\n const tipId = getUID(this.constructor.NAME).toString()\n\n tip.setAttribute('id', tipId)\n\n if (this._isAnimated()) {\n tip.classList.add(CLASS_NAME_FADE)\n }\n\n return tip\n }\n\n setContent(content) {\n this._newContent = content\n if (this._isShown()) {\n this._disposePopper()\n this.show()\n }\n }\n\n _getTemplateFactory(content) {\n if (this._templateFactory) {\n this._templateFactory.changeContent(content)\n } else {\n this._templateFactory = new TemplateFactory({\n ...this._config,\n // the `content` var has to be after `this._config`\n // to override config.content in case of popover\n content,\n extraClass: this._resolvePossibleFunction(this._config.customClass)\n })\n }\n\n return this._templateFactory\n }\n\n _getContentForTemplate() {\n return {\n [SELECTOR_TOOLTIP_INNER]: this._getTitle()\n }\n }\n\n _getTitle() {\n return this._resolvePossibleFunction(this._config.title) || this._element.getAttribute('data-bs-original-title')\n }\n\n // Private\n _initializeOnDelegatedTarget(event) {\n return this.constructor.getOrCreateInstance(event.delegateTarget, this._getDelegateConfig())\n }\n\n _isAnimated() {\n return this._config.animation || (this.tip && this.tip.classList.contains(CLASS_NAME_FADE))\n }\n\n _isShown() {\n return this.tip && this.tip.classList.contains(CLASS_NAME_SHOW)\n }\n\n _createPopper(tip) {\n const placement = execute(this._config.placement, [this, tip, this._element])\n const attachment = AttachmentMap[placement.toUpperCase()]\n return Popper.createPopper(this._element, tip, this._getPopperConfig(attachment))\n }\n\n _getOffset() {\n const { offset } = this._config\n\n if (typeof offset === 'string') {\n return offset.split(',').map(value => Number.parseInt(value, 10))\n }\n\n if (typeof offset === 'function') {\n return popperData => offset(popperData, this._element)\n }\n\n return offset\n }\n\n _resolvePossibleFunction(arg) {\n return execute(arg, [this._element])\n }\n\n _getPopperConfig(attachment) {\n const defaultBsPopperConfig = {\n placement: attachment,\n modifiers: [\n {\n name: 'flip',\n options: {\n fallbackPlacements: this._config.fallbackPlacements\n }\n },\n {\n name: 'offset',\n options: {\n offset: this._getOffset()\n }\n },\n {\n name: 'preventOverflow',\n options: {\n boundary: this._config.boundary\n }\n },\n {\n name: 'arrow',\n options: {\n element: `.${this.constructor.NAME}-arrow`\n }\n },\n {\n name: 'preSetPlacement',\n enabled: true,\n phase: 'beforeMain',\n fn: data => {\n // Pre-set Popper's placement attribute in order to read the arrow sizes properly.\n // Otherwise, Popper mixes up the width and height dimensions since the initial arrow style is for top placement\n this._getTipElement().setAttribute('data-popper-placement', data.state.placement)\n }\n }\n ]\n }\n\n return {\n ...defaultBsPopperConfig,\n ...execute(this._config.popperConfig, [defaultBsPopperConfig])\n }\n }\n\n _setListeners() {\n const triggers = this._config.trigger.split(' ')\n\n for (const trigger of triggers) {\n if (trigger === 'click') {\n EventHandler.on(this._element, this.constructor.eventName(EVENT_CLICK), this._config.selector, event => {\n const context = this._initializeOnDelegatedTarget(event)\n context.toggle()\n })\n } else if (trigger !== TRIGGER_MANUAL) {\n const eventIn = trigger === TRIGGER_HOVER ?\n this.constructor.eventName(EVENT_MOUSEENTER) :\n this.constructor.eventName(EVENT_FOCUSIN)\n const eventOut = trigger === TRIGGER_HOVER ?\n this.constructor.eventName(EVENT_MOUSELEAVE) :\n this.constructor.eventName(EVENT_FOCUSOUT)\n\n EventHandler.on(this._element, eventIn, this._config.selector, event => {\n const context = this._initializeOnDelegatedTarget(event)\n context._activeTrigger[event.type === 'focusin' ? TRIGGER_FOCUS : TRIGGER_HOVER] = true\n context._enter()\n })\n EventHandler.on(this._element, eventOut, this._config.selector, event => {\n const context = this._initializeOnDelegatedTarget(event)\n context._activeTrigger[event.type === 'focusout' ? TRIGGER_FOCUS : TRIGGER_HOVER] =\n context._element.contains(event.relatedTarget)\n\n context._leave()\n })\n }\n }\n\n this._hideModalHandler = () => {\n if (this._element) {\n this.hide()\n }\n }\n\n EventHandler.on(this._element.closest(SELECTOR_MODAL), EVENT_MODAL_HIDE, this._hideModalHandler)\n }\n\n _fixTitle() {\n const title = this._element.getAttribute('title')\n\n if (!title) {\n return\n }\n\n if (!this._element.getAttribute('aria-label') && !this._element.textContent.trim()) {\n this._element.setAttribute('aria-label', title)\n }\n\n this._element.setAttribute('data-bs-original-title', title) // DO NOT USE IT. Is only for backwards compatibility\n this._element.removeAttribute('title')\n }\n\n _enter() {\n if (this._isShown() || this._isHovered) {\n this._isHovered = true\n return\n }\n\n this._isHovered = true\n\n this._setTimeout(() => {\n if (this._isHovered) {\n this.show()\n }\n }, this._config.delay.show)\n }\n\n _leave() {\n if (this._isWithActiveTrigger()) {\n return\n }\n\n this._isHovered = false\n\n this._setTimeout(() => {\n if (!this._isHovered) {\n this.hide()\n }\n }, this._config.delay.hide)\n }\n\n _setTimeout(handler, timeout) {\n clearTimeout(this._timeout)\n this._timeout = setTimeout(handler, timeout)\n }\n\n _isWithActiveTrigger() {\n return Object.values(this._activeTrigger).includes(true)\n }\n\n _getConfig(config) {\n const dataAttributes = Manipulator.getDataAttributes(this._element)\n\n for (const dataAttribute of Object.keys(dataAttributes)) {\n if (DISALLOWED_ATTRIBUTES.has(dataAttribute)) {\n delete dataAttributes[dataAttribute]\n }\n }\n\n config = {\n ...dataAttributes,\n ...(typeof config === 'object' && config ? config : {})\n }\n config = this._mergeConfigObj(config)\n config = this._configAfterMerge(config)\n this._typeCheckConfig(config)\n return config\n }\n\n _configAfterMerge(config) {\n config.container = config.container === false ? document.body : getElement(config.container)\n\n if (typeof config.delay === 'number') {\n config.delay = {\n show: config.delay,\n hide: config.delay\n }\n }\n\n if (typeof config.title === 'number') {\n config.title = config.title.toString()\n }\n\n if (typeof config.content === 'number') {\n config.content = config.content.toString()\n }\n\n return config\n }\n\n _getDelegateConfig() {\n const config = {}\n\n for (const [key, value] of Object.entries(this._config)) {\n if (this.constructor.Default[key] !== value) {\n config[key] = value\n }\n }\n\n config.selector = false\n config.trigger = 'manual'\n\n // In the future can be replaced with:\n // const keysWithDifferentValues = Object.entries(this._config).filter(entry => this.constructor.Default[entry[0]] !== this._config[entry[0]])\n // `Object.fromEntries(keysWithDifferentValues)`\n return config\n }\n\n _disposePopper() {\n if (this._popper) {\n this._popper.destroy()\n this._popper = null\n }\n\n if (this.tip) {\n this.tip.remove()\n this.tip = null\n }\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Tooltip.getOrCreateInstance(this, config)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (typeof data[config] === 'undefined') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config]()\n })\n }\n}\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Tooltip)\n\nexport default Tooltip\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap popover.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport Tooltip from './tooltip.js'\nimport { defineJQueryPlugin } from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'popover'\n\nconst SELECTOR_TITLE = '.popover-header'\nconst SELECTOR_CONTENT = '.popover-body'\n\nconst Default = {\n ...Tooltip.Default,\n content: '',\n offset: [0, 8],\n placement: 'right',\n template: '
' +\n '
' +\n '

' +\n '
' +\n '
',\n trigger: 'click'\n}\n\nconst DefaultType = {\n ...Tooltip.DefaultType,\n content: '(null|string|element|function)'\n}\n\n/**\n * Class definition\n */\n\nclass Popover extends Tooltip {\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Overrides\n _isWithContent() {\n return this._getTitle() || this._getContent()\n }\n\n // Private\n _getContentForTemplate() {\n return {\n [SELECTOR_TITLE]: this._getTitle(),\n [SELECTOR_CONTENT]: this._getContent()\n }\n }\n\n _getContent() {\n return this._resolvePossibleFunction(this._config.content)\n }\n\n // Static\n static jQueryInterface(config) {\n return this.each(function () {\n const data = Popover.getOrCreateInstance(this, config)\n\n if (typeof config !== 'string') {\n return\n }\n\n if (typeof data[config] === 'undefined') {\n throw new TypeError(`No method named \"${config}\"`)\n }\n\n data[config]()\n })\n }\n}\n\n/**\n * jQuery\n */\n\ndefineJQueryPlugin(Popover)\n\nexport default Popover\n","/**\n * --------------------------------------------------------------------------\n * Bootstrap scrollspy.js\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n * --------------------------------------------------------------------------\n */\n\nimport BaseComponent from './base-component.js'\nimport EventHandler from './dom/event-handler.js'\nimport SelectorEngine from './dom/selector-engine.js'\nimport { defineJQueryPlugin, getElement, isDisabled, isVisible } from './util/index.js'\n\n/**\n * Constants\n */\n\nconst NAME = 'scrollspy'\nconst DATA_KEY = 'bs.scrollspy'\nconst EVENT_KEY = `.${DATA_KEY}`\nconst DATA_API_KEY = '.data-api'\n\nconst EVENT_ACTIVATE = `activate${EVENT_KEY}`\nconst EVENT_CLICK = `click${EVENT_KEY}`\nconst EVENT_LOAD_DATA_API = `load${EVENT_KEY}${DATA_API_KEY}`\n\nconst CLASS_NAME_DROPDOWN_ITEM = 'dropdown-item'\nconst CLASS_NAME_ACTIVE = 'active'\n\nconst SELECTOR_DATA_SPY = '[data-bs-spy=\"scroll\"]'\nconst SELECTOR_TARGET_LINKS = '[href]'\nconst SELECTOR_NAV_LIST_GROUP = '.nav, .list-group'\nconst SELECTOR_NAV_LINKS = '.nav-link'\nconst SELECTOR_NAV_ITEMS = '.nav-item'\nconst SELECTOR_LIST_ITEMS = '.list-group-item'\nconst SELECTOR_LINK_ITEMS = `${SELECTOR_NAV_LINKS}, ${SELECTOR_NAV_ITEMS} > ${SELECTOR_NAV_LINKS}, ${SELECTOR_LIST_ITEMS}`\nconst SELECTOR_DROPDOWN = '.dropdown'\nconst SELECTOR_DROPDOWN_TOGGLE = '.dropdown-toggle'\n\nconst Default = {\n offset: null, // TODO: v6 @deprecated, keep it for backwards compatibility reasons\n rootMargin: '0px 0px -25%',\n smoothScroll: false,\n target: null,\n threshold: [0.1, 0.5, 1]\n}\n\nconst DefaultType = {\n offset: '(number|null)', // TODO v6 @deprecated, keep it for backwards compatibility reasons\n rootMargin: 'string',\n smoothScroll: 'boolean',\n target: 'element',\n threshold: 'array'\n}\n\n/**\n * Class definition\n */\n\nclass ScrollSpy extends BaseComponent {\n constructor(element, config) {\n super(element, config)\n\n // this._element is the observablesContainer and config.target the menu links wrapper\n this._targetLinks = new Map()\n this._observableSections = new Map()\n this._rootElement = getComputedStyle(this._element).overflowY === 'visible' ? null : this._element\n this._activeTarget = null\n this._observer = null\n this._previousScrollData = {\n visibleEntryTop: 0,\n parentScrollTop: 0\n }\n this.refresh() // initialize\n }\n\n // Getters\n static get Default() {\n return Default\n }\n\n static get DefaultType() {\n return DefaultType\n }\n\n static get NAME() {\n return NAME\n }\n\n // Public\n refresh() {\n this._initializeTargetsAndObservables()\n this._maybeEnableSmoothScroll()\n\n if (this._observer) {\n this._observer.disconnect()\n } else {\n this._observer = this._getNewObserver()\n }\n\n for (const section of this._observableSections.values()) {\n this._observer.observe(section)\n }\n }\n\n dispose() {\n this._observer.disconnect()\n super.dispose()\n }\n\n // Private\n _configAfterMerge(config) {\n // TODO: on v6 target should be given explicitly & remove the {target: 'ss-target'} case\n config.target = getElement(config.target) || document.body\n\n // TODO: v6 Only for backwards compatibility reasons. Use rootMargin only\n config.rootMargin = config.offset ? `${config.offset}px 0px -30%` : config.rootMargin\n\n if (typeof config.threshold === 'string') {\n config.threshold = config.threshold.split(',').map(value => Number.parseFloat(value))\n }\n\n return config\n }\n\n _maybeEnableSmoothScroll() {\n if (!this._config.smoothScroll) {\n return\n }\n\n // unregister any previous listeners\n EventHandler.off(this._config.target, EVENT_CLICK)\n\n EventHandler.on(this._config.target, EVENT_CLICK, SELECTOR_TARGET_LINKS, event => {\n const observableSection = this._observableSections.get(event.target.hash)\n if (observableSection) {\n event.preventDefault()\n const root = this._rootElement || window\n const height = observableSection.offsetTop - this._element.offsetTop\n if (root.scrollTo) {\n root.scrollTo({ top: height, behavior: 'smooth' })\n return\n }\n\n // Chrome 60 doesn't support `scrollTo`\n root.scrollTop = height\n }\n })\n }\n\n _getNewObserver() {\n const options = {\n root: this._rootElement,\n threshold: this._config.threshold,\n rootMargin: this._config.rootMargin\n }\n\n return new IntersectionObserver(entries => this._observerCallback(entries), options)\n }\n\n // The logic of selection\n _observerCallback(entries) {\n const targetElement = entry => this._targetLinks.get(`#${entry.target.id}`)\n const activate = entry => {\n this._previousScrollData.visibleEntryTop = entry.target.offsetTop\n this._process(targetElement(entry))\n }\n\n const parentScrollTop = (this._rootElement || document.documentElement).scrollTop\n const userScrollsDown = parentScrollTop >= this._previousScrollData.parentScrollTop\n this._previousScrollData.parentScrollTop = parentScrollTop\n\n for (const entry of entries) {\n if (!entry.isIntersecting) {\n this._activeTarget = null\n this._clearActiveClass(targetElement(entry))\n\n continue\n }\n\n const entryIsLowerThanPrevious = entry.target.offsetTop >= this._previousScrollData.visibleEntryTop\n // if we are scrolling down, pick the bigger offsetTop\n if (userScrollsDown && entryIsLowerThanPrevious) {\n activate(entry)\n // if parent isn't scrolled, let's keep the first visible item, breaking the iteration\n if (!parentScrollTop) {\n return\n }\n\n continue\n }\n\n // if we are scrolling up, pick the smallest offsetTop\n if (!userScrollsDown && !entryIsLowerThanPrevious) {\n activate(entry)\n }\n }\n }\n\n _initializeTargetsAndObservables() {\n this._targetLinks = new Map()\n this._observableSections = new Map()\n\n const targetLinks = SelectorEngine.find(SELECTOR_TARGET_LINKS, this._config.target)\n\n for (const anchor of targetLinks) {\n // ensure that the anchor has an id and is not disabled\n if (!anchor.hash || isDisabled(anchor)) {\n continue\n }\n\n const observableSection = SelectorEngine.findOne(decodeURI(anchor.hash), this._element)\n\n // ensure that the observableSection exists & is visible\n if (isVisible(observableSection)) {\n this._targetLinks.set(decodeURI(anchor.hash), anchor)\n this._observableSections.set(anchor.hash, observableSection)\n }\n }\n }\n\n _process(target) {\n if (this._activeTarget === target) {\n return\n }\n\n this._clearActiveClass(this._config.target)\n this._activeTarget = target\n target.classList.add(CLASS_NAME_ACTIVE)\n this._activateParents(target)\n\n EventHandler.trigger(this._element, EVENT_ACTIVATE, { relatedTarget: target })\n }\n\n _activateParents(target) {\n // Activate dropdown parents\n if (target.classList.contains(CLASS_NAME_DROPDOWN_ITEM)) {\n SelectorEngine.findOne(SELECTOR_DROPDOWN_TOGGLE, target.closest(SELECTOR_DROPDOWN))\n .classList.add(CLASS_NAME_ACTIVE)\n return\n }\n\n for (const listGroup of SelectorEngine.parents(target, SELECTOR_NAV_LIST_GROUP)) {\n // Set triggered links parents as active\n // With both
diff --git a/dev/pkgdown.yml b/dev/pkgdown.yml index 2a56c825..df7c8cf2 100644 --- a/dev/pkgdown.yml +++ b/dev/pkgdown.yml @@ -5,7 +5,7 @@ articles: grouping: grouping.html metric-types: metric-types.html multiclass: multiclass.html -last_built: 2023-11-19T22:14Z +last_built: 2023-12-01T19:53Z urls: reference: https://yardstick.tidymodels.org/reference article: https://yardstick.tidymodels.org/articles diff --git a/dev/reference/accuracy.html b/dev/reference/accuracy.html index 0a9ddfc0..b63e17a5 100644 --- a/dev/reference/accuracy.html +++ b/dev/reference/accuracy.html @@ -1,5 +1,5 @@ -Accuracy — accuracy • yardstickAccuracy — accuracy • yardstick diff --git a/dev/reference/average_precision.html b/dev/reference/average_precision.html index 2bb6001f..8c8b5159 100644 --- a/dev/reference/average_precision.html +++ b/dev/reference/average_precision.html @@ -5,7 +5,7 @@ others say 1, others say undefined). It computes a weighted average of the precision values returned from pr_curve(), where the weights are the increase in recall from the previous -threshold. See pr_curve() for the full curve.">Area under the precision recall curve — average_precision • yardstickArea under the precision recall curve — average_precision • yardstickBalanced accuracy — bal_accuracy • yardstickBalanced accuracy — bal_accuracy • yardstick diff --git a/dev/reference/brier_class.html b/dev/reference/brier_class.html index 773dbe65..81e3e0b0 100644 --- a/dev/reference/brier_class.html +++ b/dev/reference/brier_class.html @@ -1,5 +1,5 @@ -Brier score for classification models — brier_class • yardstickBrier score for classification models — brier_class • yardstick diff --git a/dev/reference/brier_survival.html b/dev/reference/brier_survival.html index bf3fc440..93ed79ad 100644 --- a/dev/reference/brier_survival.html +++ b/dev/reference/brier_survival.html @@ -1,6 +1,6 @@ Time-Dependent Brier score for right censored data — brier_survival • yardstickTime-Dependent Brier score for right censored data — brier_survival • yardstickIntegrated Brier score for right censored data — brier_survival_integrated • yardstickIntegrated Brier score for right censored data — brier_survival_integrated • yardstickConcordance correlation coefficient — ccc • yardstickConcordance correlation coefficient — ccc • yardstick diff --git a/dev/reference/check_metric.html b/dev/reference/check_metric.html index bde8f2ac..954905e8 100644 --- a/dev/reference/check_metric.html +++ b/dev/reference/check_metric.html @@ -3,7 +3,7 @@ are useful alongside metric-summarizers for implementing new custom metrics. metric-summarizers call the metric function inside dplyr::summarise(). These functions perform checks on the inputs in -accordance with the type of metric that is used.">Developer function for checking inputs in new metrics — check_metric • yardstickDeveloper function for checking inputs in new metrics — check_metric • yardstickCosts function for poor classification — classification_cost • yardstickCosts function for poor classification — classification_cost • yardstickConcordance index for right-censored data — concordance_survival • yardstickConcordance index for right-censored data — concordance_survival • yardstick diff --git a/dev/reference/conf_mat.html b/dev/reference/conf_mat.html index 0f1b3c0a..a8c9d428 100644 --- a/dev/reference/conf_mat.html +++ b/dev/reference/conf_mat.html @@ -1,5 +1,5 @@ -Confusion Matrix for Categorical Data — conf_mat • yardstickConfusion Matrix for Categorical Data — conf_mat • yardstick diff --git a/dev/reference/demographic_parity.html b/dev/reference/demographic_parity.html index 664aa6f6..8618b857 100644 --- a/dev/reference/demographic_parity.html +++ b/dev/reference/demographic_parity.html @@ -8,7 +8,7 @@ and smallest value of detection_prevalence() across groups. Demographic parity is sometimes referred to as group fairness, disparate impact, or statistical parity. -See the "Measuring Disparity" section for details on implementation.">Demographic parity — demographic_parity • yardstickDemographic parity — demographic_parity • yardstickDetection prevalence — detection_prevalence • yardstickDetection prevalence — detection_prevalence • yardstickDeveloper helpers — developer-helpers • yardstickDeveloper helpers — developer-helpers • yardstickEqual opportunity — equal_opportunity • yardstickEqualized odds — equalized_odds • yardstickEqualized odds — equalized_odds • yardstickF Measure — f_meas • yardstickF Measure — f_meas • yardstickGain capture — gain_capture • yardstickGain capture — gain_capture • yardstickGain curve — gain_curve • yardstickGain curve — gain_curve • yardstickMulticlass Probability Predictions — hpc_cv • yardstickMulticlass Probability Predictions — hpc_cv • yardstick diff --git a/dev/reference/huber_loss.html b/dev/reference/huber_loss.html index 5c854127..3d9c3fe0 100644 --- a/dev/reference/huber_loss.html +++ b/dev/reference/huber_loss.html @@ -1,7 +1,7 @@ Huber loss — huber_loss • yardstickHuber loss — huber_loss • yardstickPsuedo-Huber Loss — huber_loss_pseudo • yardstickPsuedo-Huber Loss — huber_loss_pseudo • yardstickIndex of ideality of correlation — iic • yardstickFunction reference • yardstickFunction reference • yardstick diff --git a/dev/reference/j_index.html b/dev/reference/j_index.html index 0145f51b..afd83bce 100644 --- a/dev/reference/j_index.html +++ b/dev/reference/j_index.html @@ -1,7 +1,7 @@ J-index — j_index • yardstickJ-index — j_index • yardstickKappa — kap • yardstickKappa — kap • yardstickLift curve — lift_curve • yardstickLift curve — lift_curve • yardstickSurvival Analysis Results — lung_surv • yardstickSurvival Analysis Results — lung_surv • yardstick diff --git a/dev/reference/mae.html b/dev/reference/mae.html index 03dc4798..e62170f9 100644 --- a/dev/reference/mae.html +++ b/dev/reference/mae.html @@ -1,6 +1,6 @@ Mean absolute error — mae • yardstickMean absolute error — mae • yardstickMean absolute percent error — mape • yardstickMean absolute percent error — mape • yardstickMean absolute scaled error — mase • yardstickMean absolute scaled error — mase • yardstickMatthews correlation coefficient — mcc • yardstickMatthews correlation coefficient — mcc • yardstick diff --git a/dev/reference/metric-summarizers.html b/dev/reference/metric-summarizers.html index 23de77ab..9ef6453b 100644 --- a/dev/reference/metric-summarizers.html +++ b/dev/reference/metric-summarizers.html @@ -6,7 +6,7 @@ yardstick_remove_missing for implementing new custom metrics. These functions call the metric function inside dplyr::summarise() or dplyr::reframe() for curve_metric_summarizer(). See Custom performance metrics for more -information.">Developer function for summarizing new metrics — metric-summarizers • yardstickDeveloper function for summarizing new metrics — metric-summarizers • yardstickCombine metric functions — metric_set • yardstickCombine metric functions — metric_set • yardstickDeveloper function for summarizing new metrics — metric_summarizer • yardstickDeveloper function for summarizing new metrics — metric_summarizer • yardstickTweak a metric function — metric_tweak • yardstickDeveloper function for calling new metrics — metric_vec_template • yardstickDeveloper function for calling new metrics — metric_vec_template • yardstickGeneral Function to Estimate Performance — metrics • yardstickGeneral Function to Estimate Performance — metrics • yardstickMean log loss for multinomial data — mn_log_loss • yardstickMean log loss for multinomial data — mn_log_loss • yardstick diff --git a/dev/reference/mpe.html b/dev/reference/mpe.html index 8f96f7ec..0ac971ca 100644 --- a/dev/reference/mpe.html +++ b/dev/reference/mpe.html @@ -3,7 +3,7 @@ units. It can be used as a measure of the estimate's bias. Note that if any truth values are 0, a value of: -Inf (estimate &gt; 0), Inf (estimate &lt; 0), or NaN (estimate == 0) -is returned for mpe().">Mean percentage error — mpe • yardstickMean percentage error — mpe • yardstickMean signed deviation — msd • yardstickMean signed deviation — msd • yardstickConstruct a new metric function — new-metric • yardstickCreate groupwise metrics — new_groupwise_metric • yardstickCreate groupwise metrics — new_groupwise_metric • yardstickNegative predictive value — npv • yardstickNegative predictive value — npv • yardstickLiver Pathology Data — pathology • yardstickLiver Pathology Data — pathology • yardstick diff --git a/dev/reference/poisson_log_loss.html b/dev/reference/poisson_log_loss.html index 79936466..85eacb47 100644 --- a/dev/reference/poisson_log_loss.html +++ b/dev/reference/poisson_log_loss.html @@ -1,5 +1,5 @@ -Mean log loss for Poisson data — poisson_log_loss • yardstickMean log loss for Poisson data — poisson_log_loss • yardstick diff --git a/dev/reference/ppv.html b/dev/reference/ppv.html index 6ac2229c..d4783b8c 100644 --- a/dev/reference/ppv.html +++ b/dev/reference/ppv.html @@ -1,7 +1,7 @@ Positive predictive value — ppv • yardstickPositive predictive value — ppv • yardstickArea under the precision recall curve — pr_auc • yardstickArea under the precision recall curve — pr_auc • yardstickPrecision recall curve — pr_curve • yardstickPrecision recall curve — pr_curve • yardstickPrecision — precision • yardstickPrecision — precision • yardstickRecall — recall • yardstickRecall — recall • yardstickObjects exported from other packages — reexports • yardstickRoot mean squared error — rmse • yardstickRoot mean squared error — rmse • yardstickArea under the receiver operator curve — roc_auc • yardstickArea under the receiver operator curve — roc_auc • yardstickTime-Dependent ROC AUC for Censored Data — roc_auc_survival • yardstickTime-Dependent ROC AUC for Censored Data — roc_auc_survival • yardstickArea under the ROC curve of each class against the rest, using the a priori -class distribution — roc_aunp • yardstickArea under the ROC curve of each class against the rest, using the uniform -class distribution — roc_aunu • yardstickReceiver operator curve — roc_curve • yardstickReceiver operator curve — roc_curve • yardstickTime-Dependent ROC surve for Censored Data — roc_curve_survival • yardstickTime-Dependent ROC surve for Censored Data — roc_curve_survival • yardstickRatio of performance to deviation — rpd • yardstickRatio of performance to inter-quartile — rpiq • yardstickRatio of performance to inter-quartile — rpiq • yardstickR squared — rsq • yardstickR squared — rsq • yardstickR squared - traditional — rsq_trad • yardstickR squared - traditional — rsq_trad • yardstickSensitivity — sens • yardstickSensitivity — sens • yardstickSymmetric mean absolute percentage error — smape • yardstickSymmetric mean absolute percentage error — smape • yardstickSolubility Predictions from MARS Model — solubility_test • yardstickSolubility Predictions from MARS Model — solubility_test • yardstick diff --git a/dev/reference/spec.html b/dev/reference/spec.html index 2df21383..cc0ebc40 100644 --- a/dev/reference/spec.html +++ b/dev/reference/spec.html @@ -1,7 +1,7 @@ Specificity — spec • yardstickSpecificity — spec • yardstickSummary Statistics for Confusion Matrices — summary.conf_mat • yardstickSummary Statistics for Confusion Matrices — summary.conf_mat • yardstickTwo Class Predictions — two_class_example • yardstickTwo Class Predictions — two_class_example • yardstick diff --git a/dev/reference/yardstick-package.html b/dev/reference/yardstick-package.html index 408d2f7a..a9f1040d 100644 --- a/dev/reference/yardstick-package.html +++ b/dev/reference/yardstick-package.html @@ -1,6 +1,6 @@ yardstick: Tidy Characterizations of Model Performance — yardstick-package • yardstickyardstick: Tidy Characterizations of Model Performance — yardstick-package • yardstickDeveloper function for handling missing values in new metrics — yardstick_remove_missing • yardstick # A tibble: 3,467 × 7 #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.929 0.0653 0.00579 0.0000156 Fold01 #> 5 VF VF 0.942 0.0543 0.00381 0.00000729 Fold01 #> 6 VF VF 0.951 0.0462 0.00272 0.00000384 Fold01 #> 7 VF VF 0.914 0.0782 0.00767 0.0000354 Fold01 #> 8 VF VF 0.918 0.0744 0.00726 0.0000157 Fold01 #> 9 VF VF 0.843 0.128 0.0296 0.000192 Fold01 #> 10 VF VF 0.920 0.0728 0.00703 0.0000147 Fold01 #> # ℹ 3,457 more rows set.seed(1) hpc <- tibble(hpc_cv) %>% mutate(batch = sample(c(\"a\", \"b\"), nrow(.), replace = TRUE)) %>% select(-c(VF, F, M, L)) hpc #> # A tibble: 3,467 × 4 #> obs pred Resample batch #> #> 1 VF VF Fold01 a #> 2 VF VF Fold01 b #> 3 VF VF Fold01 a #> 4 VF VF Fold01 a #> 5 VF VF Fold01 b #> 6 VF VF Fold01 a #> 7 VF VF Fold01 a #> 8 VF VF Fold01 a #> 9 VF VF Fold01 b #> 10 VF VF Fold01 b #> # ℹ 3,457 more rows hpc %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 hpc %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699"},{"path":"https://yardstick.tidymodels.org/dev/articles/grouping.html","id":"groupwise-metrics","dir":"Articles","previous_headings":"","what":"Groupwise metrics","title":"Grouping behavior in yardstick","text":"Groupwise metrics associated data-column , passed data column, metric temporarily group column, compute values groups defined column, aggregate values computed temporary grouping back level input data’s grouping. concretely, let’s turn example pre-existing grouping data. Consider portion HPC data pertaining first resample: Suppose batches data represent two groups model performance differ. quantify degree model performance differs two groups, compute accuracy values either group separately, take difference. First, computing accuracies: Now, taking difference: Groupwise metrics encode group_by() aggregation step (case, subtraction) shown yardstick metric. can define new groupwise metric new_groupwise_metric() function: fn argument yardstick metric computed data group. name argument gives name new metric ’ve created; ’ll call “accuracy difference.” aggregate argument function defining go fn output group single numeric value. output, accuracy_diff, function subclass called metric_factory: accuracy_diff now knows take accuracy values group return difference accuracy first second result output. last thing need associate object name grouping variable pass group_by(); can pass variable name accuracy_diff : output, accuracy_diff_by_batch, yardstick metric function like : can use accuracy_diff_by_batch() metric way use accuracy(). : can also add accuracy_diff_by_batch() metric sets: Groupwise metrics group-aware. passed data grouping variables column passed first argument accuracy_diff()—case, group—accuracy_diff_by_batch() behave like yardstick metric. example: Groupwise metrics form backend fairness metrics tidymodels. learn groupwise metrics applications fairness problems, see new_groupwise_metric().","code":"hpc %>% filter(Resample == \"Fold01\") #> # A tibble: 347 × 4 #> obs pred Resample batch #> #> 1 VF VF Fold01 a #> 2 VF VF Fold01 b #> 3 VF VF Fold01 a #> 4 VF VF Fold01 a #> 5 VF VF Fold01 b #> 6 VF VF Fold01 a #> 7 VF VF Fold01 a #> 8 VF VF Fold01 a #> 9 VF VF Fold01 b #> 10 VF VF Fold01 b #> # ℹ 337 more rows acc_by_group <- hpc %>% filter(Resample == \"Fold01\") %>% group_by(batch) %>% accuracy(obs, pred) acc_by_group #> # A tibble: 2 × 4 #> batch .metric .estimator .estimate #> #> 1 a accuracy multiclass 0.713 #> 2 b accuracy multiclass 0.739 diff(c(acc_by_group$.estimate[2], acc_by_group$.estimate[1])) #> [1] -0.02518607 accuracy_diff <- new_groupwise_metric( fn = accuracy, name = \"accuracy_diff\", aggregate = function(acc_by_group) { diff(c(acc_by_group$.estimate[2], acc_by_group$.estimate[1])) } ) class(accuracy_diff) #> [1] \"metric_factory\" \"function\" accuracy_diff_by_batch <- accuracy_diff(batch) class(accuracy) #> [1] \"class_metric\" \"metric\" \"function\" class(accuracy_diff_by_batch) #> [1] \"class_metric\" \"metric\" \"function\" hpc %>% filter(Resample == \"Fold01\") %>% accuracy_diff_by_batch(obs, pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 accuracy_diff batch multiclass -0.0252 acc_ms <- metric_set(accuracy, accuracy_diff_by_batch) hpc %>% filter(Resample == \"Fold01\") %>% acc_ms(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 accuracy multiclass 0.726 NA #> 2 accuracy_diff multiclass -0.0252 batch hpc %>% group_by(Resample) %>% accuracy_diff_by_batch(obs, pred) #> # A tibble: 10 × 5 #> Resample .metric .by .estimator .estimate #> #> 1 Fold01 accuracy_diff batch multiclass -0.0252 #> 2 Fold02 accuracy_diff batch multiclass 0.106 #> 3 Fold03 accuracy_diff batch multiclass 0.0220 #> 4 Fold04 accuracy_diff batch multiclass -0.000300 #> 5 Fold05 accuracy_diff batch multiclass -0.0361 #> 6 Fold06 accuracy_diff batch multiclass 0.0153 #> 7 Fold07 accuracy_diff batch multiclass -0.0323 #> 8 Fold08 accuracy_diff batch multiclass -0.0159 #> 9 Fold09 accuracy_diff batch multiclass -0.0131 #> 10 Fold10 accuracy_diff batch multiclass -0.0255"},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"metric-types","dir":"Articles","previous_headings":"","what":"Metric types","title":"Metric types","text":"three main metric types yardstick: class, class probability, numeric. type metric standardized argument syntax, metrics return kind output (tibble 3 columns). standardization allows metrics easily grouped together used grouped data frames computing multiple resamples . five types metrics, along types inputs take. Class metrics (hard predictions) truth - factor estimate - factor Class probability metrics (soft predictions) truth - factor estimate / ... - multiple numeric columns containing class probabilities Numeric metrics truth - numeric estimate - numeric Static survival metircs truth - Surv estimate - numeric dynamic survival metrics truth - Surv ... - list data.frames, containing 3 columns .eval_time, .pred_survival, .weight_censored`","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"example","dir":"Articles","previous_headings":"","what":"Example","title":"Metric types","text":"following example, hpc_cv data set used. contains class probabilities class predictions linear discriminant analysis fit HPC data set Kuhn Johnson (2013). fit 10 fold cross-validation, predictions folds included. 1 metric, 1 resample 1 metric, 10 resamples 2 metrics, 10 resamples","code":"library(yardstick) library(dplyr) data(\"hpc_cv\") hpc_cv %>% group_by(Resample) %>% slice(1:3) #> # A tibble: 30 × 7 #> # Groups: Resample [10] #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.941 0.0544 0.00441 0.0000123 Fold02 #> 5 VF VF 0.948 0.0483 0.00347 0.00000792 Fold02 #> 6 VF VF 0.958 0.0395 0.00236 0.00000310 Fold02 #> 7 VF VF 0.939 0.0556 0.00513 0.00000790 Fold03 #> 8 VF VF 0.928 0.0642 0.00777 0.0000148 Fold03 #> 9 VF VF 0.927 0.0653 0.00786 0.0000150 Fold03 #> 10 VF VF 0.949 0.0469 0.00398 0.00000935 Fold04 #> # ℹ 20 more rows hpc_cv %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 hpc_cv %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 class_metrics <- metric_set(accuracy, kap) hpc_cv %>% group_by(Resample) %>% class_metrics(obs, estimate = pred) #> # A tibble: 20 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492"},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"metrics","dir":"Articles","previous_headings":"","what":"Metrics","title":"Metric types","text":"table metrics available yardstick, grouped type.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Multiclass averaging","text":"Classification metrics yardstick truth estimate columns factors implemented binary multiclass case. multiclass implementations use micro, macro, macro_weighted averaging applicable, metrics specialized multiclass implementations.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"macro-averaging","dir":"Articles","previous_headings":"","what":"Macro averaging","title":"Multiclass averaging","text":"Macro averaging reduces multiclass predictions multiple sets binary predictions, calculates corresponding metric binary cases, averages results together. example, consider precision binary case. \\[ Pr = \\frac{TP}{TP + FP} \\] multiclass case, levels , B, C D, macro averaging reduces problem multiple one-vs-comparisons. truth estimate columns recoded two levels , precision calculated based recoded columns, “relevant” column. process repeated 3 levels get total 4 precision values. results averaged together. formula representation looks like . k classes: \\[ Pr_{macro} = \\frac{Pr_1 + Pr_2 + \\ldots + Pr_k}{k} = Pr_1 \\frac{1}{k} + Pr_2 \\frac{1}{k} + \\ldots + Pr_k \\frac{1}{k} \\] \\(PR_1\\) precision calculated recoding multiclass predictions just class 1 . Note macro averaging, classes get equal weight contributing portion precision value total (1/4). might realistic calculation large amount class imbalance. case, weighted macro average might make sense, weights calculated frequency class truth column. \\[ Pr_{weighted-macro} = Pr_1 \\frac{\\#Obs_1}{N} + Pr_2 \\frac{\\#Obs_2}{N} + \\ldots + Pr_k \\frac{\\#Obs_k}{N} \\]","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"micro-averaging","dir":"Articles","previous_headings":"","what":"Micro averaging","title":"Multiclass averaging","text":"Micro averaging treats entire set data aggregate result, calculates 1 metric rather k metrics get averaged together. precision, works calculating true positive results class using numerator, calculating true positive false positive results class, using denominator. \\[ Pr_{micro} = \\frac{TP_1 + TP_2 + \\ldots + TP_k}{(TP_1 + TP_2 + \\ldots + TP_k) + (FP_1 + FP_2 + \\ldots + FP_k)} \\] case, rather class equal weight, observation gets equal weight. gives classes observations power.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"specialized-multiclass-implementations","dir":"Articles","previous_headings":"","what":"Specialized multiclass implementations","title":"Multiclass averaging","text":"metrics known analytical multiclass extensions, need use averaging get estimate multiclass performance. Accuracy Kappa use definitions binary counterpart, accuracy counting number correctly predicted true values total number true values, kappa linear combination two accuracy values. Matthews correlation coefficient (MCC) known multiclass generalization well, sometimes called \\(R_K\\) statistic. Refer page details. ROC AUC interesting metric intuitively makes sense perform macro averaging, computes multiclass AUC average area multiple binary ROC curves. However, loses important property ROC AUC statistic binary case insensitive class distribution. combat , multiclass metric created retains insensitivity class distribution, easy visual interpretation like macro averaging. implemented \"hand_till\" method, default metric.","code":""},{"path":"https://yardstick.tidymodels.org/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Max Kuhn. Author. Davis Vaughan. Author. Emil Hvitfeldt. Author, maintainer. . Copyright holder, funder.","code":""},{"path":"https://yardstick.tidymodels.org/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Kuhn M, Vaughan D, Hvitfeldt E (2023). yardstick: Tidy Characterizations Model Performance. R package version 1.2.0.9001, https://yardstick.tidymodels.org, https://github.com/tidymodels/yardstick.","code":"@Manual{, title = {yardstick: Tidy Characterizations of Model Performance}, author = {Max Kuhn and Davis Vaughan and Emil Hvitfeldt}, year = {2023}, note = {R package version 1.2.0.9001, https://yardstick.tidymodels.org}, url = {https://github.com/tidymodels/yardstick}, }"},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"overview","dir":"","previous_headings":"","what":"Overview","title":"Tidy Characterizations of Model Performance","text":"yardstick package estimate well models working using tidy data principles. See package webpage information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Tidy Characterizations of Model Performance","text":"install package:","code":"install.packages(\"yardstick\") # Development version: # install.packages(\"pak\") pak::pak(\"tidymodels/yardstick\")"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"two-class-metric","dir":"","previous_headings":"","what":"Two class metric","title":"Tidy Characterizations of Model Performance","text":"example, suppose create classification model predict new data set. might data looks like : can use dplyr-like syntax compute common performance characteristics model get back data frame:","code":"library(yardstick) library(dplyr) head(two_class_example) #> truth Class1 Class2 predicted #> 1 Class2 0.00359 0.996411 Class2 #> 2 Class1 0.67862 0.321379 Class1 #> 3 Class2 0.11089 0.889106 Class2 #> 4 Class1 0.73516 0.264838 Class1 #> 5 Class2 0.01624 0.983760 Class2 #> 6 Class1 0.99928 0.000725 Class1 metrics(two_class_example, truth, predicted) #> # A tibble: 2 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 # or two_class_example %>% roc_auc(truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc binary 0.939"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"multiclass-metrics","dir":"","previous_headings":"","what":"Multiclass metrics","title":"Tidy Characterizations of Model Performance","text":"classification metrics least one multiclass extension, many multiple ways calculate multiclass metrics.","code":"data(\"hpc_cv\") hpc_cv <- as_tibble(hpc_cv) hpc_cv #> # A tibble: 3,467 × 7 #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.929 0.0653 0.00579 0.0000156 Fold01 #> 5 VF VF 0.942 0.0543 0.00381 0.00000729 Fold01 #> 6 VF VF 0.951 0.0462 0.00272 0.00000384 Fold01 #> 7 VF VF 0.914 0.0782 0.00767 0.0000354 Fold01 #> 8 VF VF 0.918 0.0744 0.00726 0.0000157 Fold01 #> 9 VF VF 0.843 0.128 0.0296 0.000192 Fold01 #> 10 VF VF 0.920 0.0728 0.00703 0.0000147 Fold01 #> # ℹ 3,457 more rows # Macro averaged multiclass precision precision(hpc_cv, obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision macro 0.631 # Micro averaged multiclass precision precision(hpc_cv, obs, pred, estimator = \"micro\") #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision micro 0.709"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"calculating-metrics-on-resamples","dir":"","previous_headings":"","what":"Calculating metrics on resamples","title":"Tidy Characterizations of Model Performance","text":"multiple resamples model, can use metric grouped data frame calculate metric across resamples . calculates multiclass ROC AUC using method described Hand, Till (2001), across 10 resamples .","code":"hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc hand_till 0.813 #> 2 Fold02 roc_auc hand_till 0.817 #> 3 Fold03 roc_auc hand_till 0.869 #> 4 Fold04 roc_auc hand_till 0.849 #> 5 Fold05 roc_auc hand_till 0.811 #> 6 Fold06 roc_auc hand_till 0.836 #> 7 Fold07 roc_auc hand_till 0.825 #> 8 Fold08 roc_auc hand_till 0.846 #> 9 Fold09 roc_auc hand_till 0.828 #> 10 Fold10 roc_auc hand_till 0.812"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"autoplot-methods-for-easy-visualization","dir":"","previous_headings":"","what":"Autoplot methods for easy visualization","title":"Tidy Characterizations of Model Performance","text":"Curve based methods roc_curve(), pr_curve() gain_curve() ggplot2::autoplot() methods allow powerful easy visualization.","code":"library(ggplot2) hpc_cv %>% group_by(Resample) %>% roc_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Tidy Characterizations of Model Performance","text":"project released Contributor Code Conduct. contributing project, agree abide terms. questions discussions tidymodels packages, modeling, machine learning, please post RStudio Community. think encountered bug, please submit issue. Either way, learn create share reprex (minimal, reproducible example), clearly communicate code. Check details contributing guidelines tidymodels packages get help.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":null,"dir":"Reference","previous_headings":"","what":"Accuracy — accuracy","title":"Accuracy — accuracy","text":"Accuracy proportion data predicted correctly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Accuracy — accuracy","text":"","code":"accuracy(data, ...) # S3 method for data.frame accuracy(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) accuracy_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Accuracy — accuracy","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Accuracy — accuracy","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. accuracy_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Accuracy — accuracy","text":"Accuracy extends naturally multiclass scenarios. , macro micro averaging implemented.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Accuracy — accuracy","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Accuracy — accuracy","text":"","code":"library(dplyr) #> #> Attaching package: ‘dplyr’ #> The following objects are masked from ‘package:stats’: #> #> filter, lag #> The following objects are masked from ‘package:base’: #> #> intersect, setdiff, setequal, union data(\"two_class_example\") data(\"hpc_cv\") # Two class accuracy(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 # Multiclass # accuracy() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 # Groups are respected hpc_cv %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699"},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the precision recall curve — average_precision","title":"Area under the precision recall curve — average_precision","text":"average_precision() alternative pr_auc() avoids ambiguity value precision recall == 0 yet false positive values (say 0, others say 1, others say undefined). computes weighted average precision values returned pr_curve(), weights increase recall previous threshold. See pr_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the precision recall curve — average_precision","text":"","code":"average_precision(data, ...) # S3 method for data.frame average_precision( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) average_precision_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the precision recall curve — average_precision","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the precision recall curve — average_precision","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. average_precision_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Area under the precision recall curve — average_precision","text":"computation average precision weighted average precision values. Assuming n rows returned pr_curve(), sum 2 n, multiplying precision value p_i increase recall previous threshold, r_i - r_(-1). $$AP = \\sum (r_{} - r_{-1}) * p_i$$ summing 2 n, precision value p_1 never used. pr_curve() returns value p_1, technically undefined tp / (tp + fp) tp = 0 fp = 0. common convention use 1 p_1, metric nice property avoiding ambiguity. hand, r_1 well defined long events (p), tp / p tp = 0, r_1 = 0. p_1 defined 1, average_precision() roc_auc() values often close one another.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the precision recall curve — average_precision","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the precision recall curve — average_precision","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the precision recall curve — average_precision","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. average_precision(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision binary 0.947 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% average_precision(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision macro 0.617 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% average_precision(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision macro 0.617 # Groups are respected hpc_cv %>% group_by(Resample) %>% average_precision(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 average_precision macro 0.617 #> 2 Fold02 average_precision macro 0.625 #> 3 Fold03 average_precision macro 0.699 #> 4 Fold04 average_precision macro 0.685 #> 5 Fold05 average_precision macro 0.625 #> 6 Fold06 average_precision macro 0.656 #> 7 Fold07 average_precision macro 0.617 #> 8 Fold08 average_precision macro 0.659 #> 9 Fold09 average_precision macro 0.632 #> 10 Fold10 average_precision macro 0.611 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% average_precision(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 average_precision macro_weighted 0.750 #> 2 Fold02 average_precision macro_weighted 0.745 #> 3 Fold03 average_precision macro_weighted 0.794 #> 4 Fold04 average_precision macro_weighted 0.757 #> 5 Fold05 average_precision macro_weighted 0.740 #> 6 Fold06 average_precision macro_weighted 0.747 #> 7 Fold07 average_precision macro_weighted 0.751 #> 8 Fold08 average_precision macro_weighted 0.759 #> 9 Fold09 average_precision macro_weighted 0.714 #> 10 Fold10 average_precision macro_weighted 0.742 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") average_precision_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.6173363"},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":null,"dir":"Reference","previous_headings":"","what":"Balanced accuracy — bal_accuracy","title":"Balanced accuracy — bal_accuracy","text":"Balanced accuracy computed average sens() spec().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Balanced accuracy — bal_accuracy","text":"","code":"bal_accuracy(data, ...) # S3 method for data.frame bal_accuracy( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) bal_accuracy_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Balanced accuracy — bal_accuracy","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Balanced accuracy — bal_accuracy","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. bal_accuracy_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Balanced accuracy — bal_accuracy","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Balanced accuracy — bal_accuracy","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Balanced accuracy — bal_accuracy","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Balanced accuracy — bal_accuracy","text":"","code":"# Two class data(\"two_class_example\") bal_accuracy(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 bal_accuracy binary 0.837 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% bal_accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 bal_accuracy macro 0.717 # Groups are respected hpc_cv %>% group_by(Resample) %>% bal_accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 bal_accuracy macro 0.717 #> 2 Fold02 bal_accuracy macro 0.711 #> 3 Fold03 bal_accuracy macro 0.767 #> 4 Fold04 bal_accuracy macro 0.724 #> 5 Fold05 bal_accuracy macro 0.715 #> 6 Fold06 bal_accuracy macro 0.707 #> 7 Fold07 bal_accuracy macro 0.699 #> 8 Fold08 bal_accuracy macro 0.734 #> 9 Fold09 bal_accuracy macro 0.717 #> 10 Fold10 bal_accuracy macro 0.706 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% bal_accuracy(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 bal_accuracy macro_weighted 0.771 #> 2 Fold02 bal_accuracy macro_weighted 0.763 #> 3 Fold03 bal_accuracy macro_weighted 0.799 #> 4 Fold04 bal_accuracy macro_weighted 0.758 #> 5 Fold05 bal_accuracy macro_weighted 0.762 #> 6 Fold06 bal_accuracy macro_weighted 0.746 #> 7 Fold07 bal_accuracy macro_weighted 0.733 #> 8 Fold08 bal_accuracy macro_weighted 0.768 #> 9 Fold09 bal_accuracy macro_weighted 0.734 #> 10 Fold10 bal_accuracy macro_weighted 0.750 # Vector version bal_accuracy_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8366167 # Making Class2 the \"relevant\" level bal_accuracy_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8366167"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":null,"dir":"Reference","previous_headings":"","what":"Brier score for classification models — brier_class","title":"Brier score for classification models — brier_class","text":"Compute Brier score classification model.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Brier score for classification models — brier_class","text":"","code":"brier_class(data, ...) # S3 method for data.frame brier_class(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_class_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Brier score for classification models — brier_class","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Brier score for classification models — brier_class","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. brier_class_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Brier score for classification models — brier_class","text":"Brier score analogous mean squared error regression models. difference binary indicator class corresponding class probability squared averaged. function uses convention Kruppa et al (2014) divides result two. Smaller values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Brier score for classification models — brier_class","text":"Brier scores can computed way number classes. , averaging types supported.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Brier score for classification models — brier_class","text":"Kruppa, J., Liu, Y., Diener, H.-C., Holste, T., Weimar, C., Koonig, . R., Ziegler, . (2014) Probability estimation machine learning methods dichotomous multicategory outcome: Applications. Biometrical Journal, 56 (4): 564-583.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Brier score for classification models — brier_class","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Brier score for classification models — brier_class","text":"","code":"# Two class data(\"two_class_example\") brier_class(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_class binary 0.106 # Multiclass library(dplyr) data(hpc_cv) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% brier_class(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_class multiclass 0.202 # Groups are respected hpc_cv %>% group_by(Resample) %>% brier_class(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 brier_class multiclass 0.202 #> 2 Fold02 brier_class multiclass 0.215 #> 3 Fold03 brier_class multiclass 0.177 #> 4 Fold04 brier_class multiclass 0.204 #> 5 Fold05 brier_class multiclass 0.213 #> 6 Fold06 brier_class multiclass 0.214 #> 7 Fold07 brier_class multiclass 0.221 #> 8 Fold08 brier_class multiclass 0.209 #> 9 Fold09 brier_class multiclass 0.235 #> 10 Fold10 brier_class multiclass 0.218"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent Brier score for right censored data — brier_survival","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"Compute time-dependent Brier score right censored data, mean squared error time point .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"","code":"brier_survival(data, ...) # S3 method for data.frame brier_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_survival_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. brier_survival_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes Brier score. account censoring, inverse probability censoring weights (IPCW) used calculations. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument. Smaller values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"E. Graf, C. Schmoor, W. Sauerbrei, M. Schumacher, “Assessment comparison prognostic classification schemes survival data,” Statistics Medicine, vol. 18, . 17-18, pp. 2529–2545, 1999.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"","code":"library(dplyr) lung_surv %>% brier_survival( truth = surv_obj, .pred ) #> # A tibble: 5 × 4 #> .metric .estimator .eval_time .estimate #> #> 1 brier_survival standard 100 0.109 #> 2 brier_survival standard 200 0.194 #> 3 brier_survival standard 300 0.219 #> 4 brier_survival standard 400 0.222 #> 5 brier_survival standard 500 0.197"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":null,"dir":"Reference","previous_headings":"","what":"Integrated Brier score for right censored data — brier_survival_integrated","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"Compute integrated Brier score right censored data, overall calculation model performance values .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"","code":"brier_survival_integrated(data, ...) # S3 method for data.frame brier_survival_integrated(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_survival_integrated_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. brier_survival_integrated_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"integrated time-dependent brier score calculated \"area curve\" fashion. brier score calculated value .eval_time. area calculated via trapezoidal rule. area divided largest value .eval_time bring scale traditional brier score. Smaller values score associated better model performance. formulation takes survival probability predictions one specific evaluation times , time, computes Brier score. account censoring, inverse probability censoring weights (IPCW) used calculations. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"E. Graf, C. Schmoor, W. Sauerbrei, M. Schumacher, “Assessment comparison prognostic classification schemes survival data,” Statistics Medicine, vol. 18, . 17-18, pp. 2529–2545, 1999.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"","code":"library(dplyr) lung_surv %>% brier_survival_integrated( truth = surv_obj, .pred ) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_survival_integrated standard 0.158"},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":null,"dir":"Reference","previous_headings":"","what":"Concordance correlation coefficient — ccc","title":"Concordance correlation coefficient — ccc","text":"Calculate concordance correlation coefficient.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Concordance correlation coefficient — ccc","text":"","code":"ccc(data, ...) # S3 method for data.frame ccc( data, truth, estimate, bias = FALSE, na_rm = TRUE, case_weights = NULL, ... ) ccc_vec(truth, estimate, bias = FALSE, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Concordance correlation coefficient — ccc","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. bias logical; biased estimate variance used (Lin (1989))? na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Concordance correlation coefficient — ccc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. ccc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Concordance correlation coefficient — ccc","text":"ccc() metric consistency/correlation accuracy, metrics rmse() strictly accuracy metrics rsq() strictly consistency/correlation","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Concordance correlation coefficient — ccc","text":"Lin, L. (1989). concordance correlation coefficient evaluate reproducibility. Biometrics, 45 (1), 255-268. Nickerson, C. (1997). note \"concordance correlation coefficient evaluate reproducibility\". Biometrics, 53(4), 1503-1507.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Concordance correlation coefficient — ccc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Concordance correlation coefficient — ccc","text":"","code":"# Supply truth and predictions as bare column names ccc(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ccc standard 0.937 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% ccc(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 ccc standard 0.935 #> 2 10 ccc standard 0.937 #> 3 2 ccc standard 0.943 #> 4 3 ccc standard 0.956 #> 5 4 ccc standard 0.944 #> 6 5 ccc standard 0.925 #> 7 6 ccc standard 0.933 #> 8 7 ccc standard 0.922 #> 9 8 ccc standard 0.955 #> 10 9 ccc standard 0.940 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.939"},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for checking inputs in new metrics — check_metric","title":"Developer function for checking inputs in new metrics — check_metric","text":"check_numeric_metric(), check_class_metric(), check_prob_metric() useful alongside metric-summarizers implementing new custom metrics. metric-summarizers call metric function inside dplyr::summarise(). functions perform checks inputs accordance type metric used.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for checking inputs in new metrics — check_metric","text":"","code":"check_numeric_metric(truth, estimate, case_weights, call = caller_env()) check_class_metric( truth, estimate, case_weights, estimator, call = caller_env() ) check_prob_metric( truth, estimate, case_weights, estimator, call = caller_env() ) check_dynamic_survival_metric( truth, estimate, case_weights, call = caller_env() ) check_static_survival_metric( truth, estimate, case_weights, call = caller_env() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for checking inputs in new metrics — check_metric","text":"truth realized vector truth. check_numeric_metric(), numeric vector. check_class_metric(), factor. check_prob_metric(), factor. check_dynamic_survival_metric(), Surv object. check_static_survival_metric(), Surv object. estimate realized estimate result. check_numeric_metric(), numeric vector. check_class_metric(), factor. check_prob_metric(), numeric vector binary truth, numeric matrix multic-class truth. check_dynamic_survival_metric(), list-column data.frames. check_static_survival_metric(), numeric vector. case_weights realized case weights, numeric vector. must length truth. call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. estimator can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":null,"dir":"Reference","previous_headings":"","what":"Costs function for poor classification — classification_cost","title":"Costs function for poor classification — classification_cost","text":"classification_cost() calculates cost poor prediction based user-defined costs. costs multiplied estimated class probabilities mean cost returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Costs function for poor classification — classification_cost","text":"","code":"classification_cost(data, ...) # S3 method for data.frame classification_cost( data, truth, ..., costs = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) classification_cost_vec( truth, estimate, costs = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Costs function for poor classification — classification_cost","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. costs data frame columns \"truth\", \"estimate\", \"cost\". \"truth\" \"estimate\" character columns containing unique combinations levels truth factor. \"costs\" numeric column representing cost applied \"estimate\" predicted, true result \"truth\". often case \"truth\" == \"estimate\", cost zero (penalty correct predictions). combinations levels truth missing, costs assumed zero. NULL, equal costs used, applying cost 0 correct predictions, cost 1 incorrect predictions. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Costs function for poor classification — classification_cost","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. class_cost_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Costs function for poor classification — classification_cost","text":"example, suppose three classes: \"\", \"B\", \"C\". Suppose truly \"\" observation class probabilities = 0.3 / B = 0.3 / C = 0.4. Suppose , true result class \"\", costs class = 0 / B = 5 / C = 10, penalizing probability incorrectly predicting \"C\" predicting \"B\". cost prediction 0.3 * 0 + 0.3 * 5 + 0.4 * 10. calculation done sample individual costs averaged.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Costs function for poor classification — classification_cost","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Costs function for poor classification — classification_cost","text":"","code":"library(dplyr) # --------------------------------------------------------------------------- # Two class example data(two_class_example) # Assuming `Class1` is our \"event\", this penalizes false positives heavily costs1 <- tribble( ~truth, ~estimate, ~cost, \"Class1\", \"Class2\", 1, \"Class2\", \"Class1\", 2 ) # Assuming `Class1` is our \"event\", this penalizes false negatives heavily costs2 <- tribble( ~truth, ~estimate, ~cost, \"Class1\", \"Class2\", 2, \"Class2\", \"Class1\", 1 ) classification_cost(two_class_example, truth, Class1, costs = costs1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost binary 0.288 classification_cost(two_class_example, truth, Class1, costs = costs2) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost binary 0.260 # --------------------------------------------------------------------------- # Multiclass data(hpc_cv) # Define cost matrix from Kuhn and Johnson (2013) hpc_costs <- tribble( ~estimate, ~truth, ~cost, \"VF\", \"VF\", 0, \"VF\", \"F\", 1, \"VF\", \"M\", 5, \"VF\", \"L\", 10, \"F\", \"VF\", 1, \"F\", \"F\", 0, \"F\", \"M\", 5, \"F\", \"L\", 5, \"M\", \"VF\", 1, \"M\", \"F\", 1, \"M\", \"M\", 0, \"M\", \"L\", 1, \"L\", \"VF\", 1, \"L\", \"F\", 1, \"L\", \"M\", 1, \"L\", \"L\", 0 ) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% classification_cost(obs, VF:L, costs = hpc_costs) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost multiclass 0.779 # Groups are respected hpc_cv %>% group_by(Resample) %>% classification_cost(obs, VF:L, costs = hpc_costs) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 classification_cost multiclass 0.779 #> 2 Fold02 classification_cost multiclass 0.735 #> 3 Fold03 classification_cost multiclass 0.654 #> 4 Fold04 classification_cost multiclass 0.754 #> 5 Fold05 classification_cost multiclass 0.777 #> 6 Fold06 classification_cost multiclass 0.737 #> 7 Fold07 classification_cost multiclass 0.743 #> 8 Fold08 classification_cost multiclass 0.749 #> 9 Fold09 classification_cost multiclass 0.760 #> 10 Fold10 classification_cost multiclass 0.771"},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Concordance index for right-censored data — concordance_survival","title":"Concordance index for right-censored data — concordance_survival","text":"Compute Concordance index right-censored data","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Concordance index for right-censored data — concordance_survival","text":"","code":"concordance_survival(data, ...) # S3 method for data.frame concordance_survival( data, truth, estimate, na_rm = TRUE, case_weights = NULL, ... ) concordance_survival_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Concordance index for right-censored data — concordance_survival","text":"data data.frame containing columns specified truth .... ... Currently used. truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. estimate column identifier predicted time, numeric variables. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Concordance index for right-censored data — concordance_survival","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. concordance_survival_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Concordance index for right-censored data — concordance_survival","text":"concordance index defined proportion comparable pairs predictions outcomes concordant. Two observations comparable : observations experienced event (different times), observation shorter observed survival time experienced event, case event-free subject “outlived” . pair comparable experienced events time. Concordance intuitively means two samples ordered correctly model. specifically, two samples concordant, one higher estimated risk score shorter actual survival time. Larger values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Concordance index for right-censored data — concordance_survival","text":"Harrell, F.E., Califf, R.M., Pryor, D.B., Lee, K.L., Rosati, R., “Multivariable prognostic models: issues developing models, evaluating assumptions adequacy, measuring reducing errors”, Statistics Medicine, 15(4), 361-87, 1996.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Concordance index for right-censored data — concordance_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Concordance index for right-censored data — concordance_survival","text":"","code":"concordance_survival( data = lung_surv, truth = surv_obj, estimate = .pred_time ) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 concordance_survival standard 0.637"},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":null,"dir":"Reference","previous_headings":"","what":"Confusion Matrix for Categorical Data — conf_mat","title":"Confusion Matrix for Categorical Data — conf_mat","text":"Calculates cross-tabulation observed predicted classes.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Confusion Matrix for Categorical Data — conf_mat","text":"","code":"conf_mat(data, ...) # S3 method for data.frame conf_mat( data, truth, estimate, dnn = c(\"Prediction\", \"Truth\"), case_weights = NULL, ... ) # S3 method for conf_mat tidy(x, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Confusion Matrix for Categorical Data — conf_mat","text":"data data frame base::table(). ... used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. dnn character vector dimnames table. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. x conf_mat object.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Confusion Matrix for Categorical Data — conf_mat","text":"conf_mat() produces object class conf_mat. contains table objects. tidy.conf_mat() generates tibble columns name (cell identifier) value (cell count). used grouped data frame, conf_mat() returns tibble containing columns groups along conf_mat, list-column element conf_mat object.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Confusion Matrix for Categorical Data — conf_mat","text":"conf_mat() objects, broom tidy() method created collapses cell counts cell data frame easy manipulation. also summary() method computes various classification metrics . See summary.conf_mat() ggplot2::autoplot() method quickly visualizing matrix. heatmap mosaic type implemented. function requires factors exactly levels.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Confusion Matrix for Categorical Data — conf_mat","text":"","code":"library(dplyr) data(\"hpc_cv\") # The confusion matrix from a single assessment set (i.e. fold) cm <- hpc_cv %>% filter(Resample == \"Fold01\") %>% conf_mat(obs, pred) cm #> Truth #> Prediction VF F M L #> VF 166 33 8 1 #> F 11 71 24 7 #> M 0 3 5 3 #> L 0 1 4 10 # Now compute the average confusion matrix across all folds in # terms of the proportion of the data contained in each cell. # First get the raw cell counts per fold using the `tidy` method library(tidyr) cells_per_resample <- hpc_cv %>% group_by(Resample) %>% conf_mat(obs, pred) %>% mutate(tidied = lapply(conf_mat, tidy)) %>% unnest(tidied) # Get the totals per resample counts_per_resample <- hpc_cv %>% group_by(Resample) %>% summarize(total = n()) %>% left_join(cells_per_resample, by = \"Resample\") %>% # Compute the proportions mutate(prop = value / total) %>% group_by(name) %>% # Average summarize(prop = mean(prop)) counts_per_resample #> # A tibble: 16 × 2 #> name prop #> #> 1 cell_1_1 0.467 #> 2 cell_1_2 0.107 #> 3 cell_1_3 0.0185 #> 4 cell_1_4 0.00259 #> 5 cell_2_1 0.0407 #> 6 cell_2_2 0.187 #> 7 cell_2_3 0.0632 #> 8 cell_2_4 0.0173 #> 9 cell_3_1 0.00173 #> 10 cell_3_2 0.00692 #> 11 cell_3_3 0.0228 #> 12 cell_3_4 0.00807 #> 13 cell_4_1 0.000575 #> 14 cell_4_2 0.0104 #> 15 cell_4_3 0.0144 #> 16 cell_4_4 0.0320 # Now reshape these into a matrix mean_cmat <- matrix(counts_per_resample$prop, byrow = TRUE, ncol = 4) rownames(mean_cmat) <- levels(hpc_cv$obs) colnames(mean_cmat) <- levels(hpc_cv$obs) round(mean_cmat, 3) #> VF F M L #> VF 0.467 0.107 0.018 0.003 #> F 0.041 0.187 0.063 0.017 #> M 0.002 0.007 0.023 0.008 #> L 0.001 0.010 0.014 0.032 # The confusion matrix can quickly be visualized using autoplot() library(ggplot2) autoplot(cm, type = \"mosaic\") autoplot(cm, type = \"heatmap\")"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":null,"dir":"Reference","previous_headings":"","what":"Demographic parity — demographic_parity","title":"Demographic parity — demographic_parity","text":"Demographic parity satisfied model's predictions predicted positive rate across groups. value 0 indicates parity across groups. Note definition depend true outcome; truth argument included outputted metrics consistency. demographic_parity() calculated difference largest smallest value detection_prevalence() across groups. Demographic parity sometimes referred group fairness, disparate impact, statistical parity. See \"Measuring Disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Demographic parity — demographic_parity","text":"","code":"demographic_parity(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Demographic parity — demographic_parity","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Demographic parity — demographic_parity","text":"function outputs yardstick fairness metric function. Given grouping variable , demographic_parity() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set detection_prevalence() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Demographic parity — demographic_parity","text":"default, function takes difference range detection_prevalence() .estimates across groups. , maximum pair-wise disparity groups return value demographic_parity()'s .estimate. finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output detection_prevalence() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted demographic_parity_2() context.","code":"# the actual default `aggregate` is: diff_range <- function(x, ...) {diff(range(x$.estimate))} demographic_parity_2 <- new_groupwise_metric( fn = \\code{\\link[=detection_prevalence]{detection_prevalence()}}, name = \"demographic_parity_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Demographic parity — demographic_parity","text":"Agarwal, ., Beygelzimer, ., Dudik, M., Langford, J., & Wallach, H. (2018). \"Reductions Approach Fair Classification.\" Proceedings 35th International Conference Machine Learning, Proceedings Machine Learning Research. 80:60-69. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Demographic parity — demographic_parity","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `demographic_parity()` by Resample m_set <- metric_set(demographic_parity(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 demographic_parity Resample macro 2.78e-17 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, demographic_parity(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 5.60e- 1 NA #> 2 demographic_parity macro 2.78e-17 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":null,"dir":"Reference","previous_headings":"","what":"Detection prevalence — detection_prevalence","title":"Detection prevalence — detection_prevalence","text":"Detection prevalence defined number predicted positive events (true positive false positive) divided total number predictions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Detection prevalence — detection_prevalence","text":"","code":"detection_prevalence(data, ...) # S3 method for data.frame detection_prevalence( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) detection_prevalence_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Detection prevalence — detection_prevalence","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Detection prevalence — detection_prevalence","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. detection_prevalence_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Detection prevalence — detection_prevalence","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Detection prevalence — detection_prevalence","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Detection prevalence — detection_prevalence","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Detection prevalence — detection_prevalence","text":"","code":"# Two class data(\"two_class_example\") detection_prevalence(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 detection_prevalence binary 0.554 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% detection_prevalence(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 detection_prevalence macro 0.25 # Groups are respected hpc_cv %>% group_by(Resample) %>% detection_prevalence(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 detection_prevalence macro 0.25 #> 2 Fold02 detection_prevalence macro 0.25 #> 3 Fold03 detection_prevalence macro 0.25 #> 4 Fold04 detection_prevalence macro 0.25 #> 5 Fold05 detection_prevalence macro 0.25 #> 6 Fold06 detection_prevalence macro 0.25 #> 7 Fold07 detection_prevalence macro 0.25 #> 8 Fold08 detection_prevalence macro 0.25 #> 9 Fold09 detection_prevalence macro 0.25 #> 10 Fold10 detection_prevalence macro 0.25 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% detection_prevalence(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 detection_prevalence macro_weighted 0.413 #> 2 Fold02 detection_prevalence macro_weighted 0.409 #> 3 Fold03 detection_prevalence macro_weighted 0.404 #> 4 Fold04 detection_prevalence macro_weighted 0.411 #> 5 Fold05 detection_prevalence macro_weighted 0.407 #> 6 Fold06 detection_prevalence macro_weighted 0.411 #> 7 Fold07 detection_prevalence macro_weighted 0.405 #> 8 Fold08 detection_prevalence macro_weighted 0.406 #> 9 Fold09 detection_prevalence macro_weighted 0.402 #> 10 Fold10 detection_prevalence macro_weighted 0.408 # Vector version detection_prevalence_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.554 # Making Class2 the \"relevant\" level detection_prevalence_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.446"},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer helpers — developer-helpers","title":"Developer helpers — developer-helpers","text":"Helpers used alongside check_metric, yardstick_remove_missing metric summarizers creating new metrics. See Custom performance metrics information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer helpers — developer-helpers","text":"","code":"dots_to_estimate(data, ...) get_weights(data, estimator) finalize_estimator( x, estimator = NULL, metric_class = \"default\", call = caller_env() ) finalize_estimator_internal( metric_dispatcher, x, estimator, call = caller_env() ) validate_estimator(estimator, estimator_override = NULL, call = caller_env())"},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer helpers — developer-helpers","text":"data table truth values columns predicted values rows. ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. estimator Either NULL auto-selection, single character type estimator use. x column used autoselect estimator. generally truth column, can also table metric table methods. metric_class single character name metric autoselect estimator . match method name created finalize_estimator_internal(). call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. metric_dispatcher simple dummy object class provided metric_class. created passed along . estimator_override character vector overriding default allowed estimator list c(\"binary\", \"macro\", \"micro\", \"macro_weighted\"). Set classification estimator support methods.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"dots-gt-estimate","dir":"Reference","previous_headings":"","what":"Dots -> Estimate","title":"Developer helpers — developer-helpers","text":"dots_to_estimate() useful class probability metrics take ... rather estimate argument. constructs either single name 1 input provided ... constructs quosure expression constructs matrix many columns provided .... eventually evaluated summarise() call metric-summarizers evaluate either vector matrix use underlying vector functions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"weight-calculation","dir":"Reference","previous_headings":"","what":"Weight Calculation","title":"Developer helpers — developer-helpers","text":"get_weights() accepts confusion matrix estimator type \"macro\", \"micro\", \"macro_weighted\" returns correct weights. useful creating multiclass metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"estimator-selection","dir":"Reference","previous_headings":"","what":"Estimator Selection","title":"Developer helpers — developer-helpers","text":"finalize_estimator() engine auto-selection estimator based type x. Generally x truth column. function called vector method metric. finalize_estimator_internal() S3 generic extend metric implement following estimator types: \"binary\", \"macro\", \"micro\", \"macro_weighted\". metric support , default version finalize_estimator_internal() autoselect estimator appropriately. need create method, take form: finalize_estimator_internal.metric_name. method finalize_estimator_internal() two things: estimator NULL, autoselect estimator based type x return single character estimator. estimator NULL, validate allowed estimator metric return . using default finalize_estimator_internal(), estimator selected using following heuristics: estimator NULL, validated returned immediately auto-selection needed. x : factor - \"binary\" returned 2 levels, otherwise \"macro\" returned. numeric - \"binary\" returned. table - \"binary\" returned 2 columns, otherwise \"macro\" returned. useful table methods. matrix - \"macro\" returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"estimator-validation","dir":"Reference","previous_headings":"","what":"Estimator Validation","title":"Developer helpers — developer-helpers","text":"validate_estimator() called metric specific method finalize_estimator_internal() ensures user provided estimator right format one allowed values.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":null,"dir":"Reference","previous_headings":"","what":"Equal opportunity — equal_opportunity","title":"Equal opportunity — equal_opportunity","text":"Equal opportunity satisfied model's predictions true positive false negative rates across protected groups. value 0 indicates parity across groups. equal_opportunity() calculated difference largest smallest value sens() across groups. Equal opportunity sometimes referred equality opportunity. See \"Measuring Disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Equal opportunity — equal_opportunity","text":"","code":"equal_opportunity(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Equal opportunity — equal_opportunity","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Equal opportunity — equal_opportunity","text":"function outputs yardstick fairness metric function. Given grouping variable , equal_opportunity() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set sens() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Equal opportunity — equal_opportunity","text":"default, function takes difference range sens() .estimates across groups. , maximum pair-wise disparity groups return value equal_opportunity()'s .estimate. finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output sens() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted equal_opportunity_2() context.","code":"# the actual default `aggregate` is: diff_range <- function(x, ...) {diff(range(x$.estimate))} equal_opportunity_2 <- new_groupwise_metric( fn = \\code{\\link[=sens]{sens()}}, name = \"equal_opportunity_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Equal opportunity — equal_opportunity","text":"Hardt, M., Price, E., & Srebro, N. (2016). \"Equality opportunity supervised learning\". Advances neural information processing systems, 29. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Equal opportunity — equal_opportunity","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `equal_opportunity()` by Resample m_set <- metric_set(equal_opportunity(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 equal_opportunity Resample macro 0.103 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, equal_opportunity(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 0.560 NA #> 2 equal_opportunity macro 0.103 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":null,"dir":"Reference","previous_headings":"","what":"Equalized odds — equalized_odds","title":"Equalized odds — equalized_odds","text":"Equalized odds satisfied model's predictions false positive, true positive, false negative, true negative rates across protected groups. value 0 indicates parity across groups. default, function takes maximum difference range sens() spec() .estimates across groups. , maximum pair-wise disparity sens() spec() groups return value equalized_odds()'s .estimate. Equalized odds sometimes referred conditional procedure accuracy equality disparate mistreatment. See \"Measuring disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Equalized odds — equalized_odds","text":"","code":"equalized_odds(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Equalized odds — equalized_odds","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Equalized odds — equalized_odds","text":"function outputs yardstick fairness metric function. Given grouping variable , equalized_odds() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set sens() spec() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Equalized odds — equalized_odds","text":"finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output sens() spec() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted equalized_odds_2() context.","code":"# see yardstick:::max_positive_rate_diff for the actual `aggregate()` diff_range <- function(x, ...) {diff(range(x$.estimate))} equalized_odds_2 <- new_groupwise_metric( fn = metric_set(sens, spec), name = \"equalized_odds_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Equalized odds — equalized_odds","text":"Agarwal, ., Beygelzimer, ., Dudik, M., Langford, J., & Wallach, H. (2018). \"Reductions Approach Fair Classification.\" Proceedings 35th International Conference Machine Learning, Proceedings Machine Learning Research. 80:60-69. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Equalized odds — equalized_odds","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `equalized_odds()` by Resample m_set <- metric_set(equalized_odds(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 equalized_odds Resample macro 0.103 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, equalized_odds(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 0.560 NA #> 2 equalized_odds macro 0.103 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":null,"dir":"Reference","previous_headings":"","what":"F Measure — f_meas","title":"F Measure — f_meas","text":"functions calculate f_meas() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions recall() precision().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"F Measure — f_meas","text":"","code":"f_meas(data, ...) # S3 method for data.frame f_meas( data, truth, estimate, beta = 1, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) f_meas_vec( truth, estimate, beta = 1, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"F Measure — f_meas","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. beta numeric value used weight precision recall. value 1 traditionally used corresponds harmonic mean two values values weight recall beta times important precision. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"F Measure — f_meas","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. f_meas_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"F Measure — f_meas","text":"measure \"F\" combination precision recall (see ).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"F Measure — f_meas","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"F Measure — f_meas","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"F Measure — f_meas","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"F Measure — f_meas","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"F Measure — f_meas","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"F Measure — f_meas","text":"","code":"# Two class data(\"two_class_example\") f_meas(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 f_meas binary 0.849 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% f_meas(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 f_meas macro 0.563 # Groups are respected hpc_cv %>% group_by(Resample) %>% f_meas(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 f_meas macro 0.563 #> 2 Fold02 f_meas macro 0.542 #> 3 Fold03 f_meas macro 0.641 #> 4 Fold04 f_meas macro 0.593 #> 5 Fold05 f_meas macro 0.570 #> 6 Fold06 f_meas macro 0.554 #> 7 Fold07 f_meas macro 0.516 #> 8 Fold08 f_meas macro 0.601 #> 9 Fold09 f_meas macro 0.555 #> 10 Fold10 f_meas macro 0.560 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% f_meas(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 f_meas macro_weighted 0.696 #> 2 Fold02 f_meas macro_weighted 0.684 #> 3 Fold03 f_meas macro_weighted 0.739 #> 4 Fold04 f_meas macro_weighted 0.689 #> 5 Fold05 f_meas macro_weighted 0.692 #> 6 Fold06 f_meas macro_weighted 0.673 #> 7 Fold07 f_meas macro_weighted 0.646 #> 8 Fold08 f_meas macro_weighted 0.701 #> 9 Fold09 f_meas macro_weighted 0.652 #> 10 Fold10 f_meas macro_weighted 0.680 # Vector version f_meas_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8485981 # Making Class2 the \"relevant\" level f_meas_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8258065"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":null,"dir":"Reference","previous_headings":"","what":"Gain capture — gain_capture","title":"Gain capture — gain_capture","text":"gain_capture() measure performance similar AUC calculation, applied gain curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Gain capture — gain_capture","text":"","code":"gain_capture(data, ...) # S3 method for data.frame gain_capture( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) gain_capture_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Gain capture — gain_capture","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Gain capture — gain_capture","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. gain_capture_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Gain capture — gain_capture","text":"gain_capture() calculates area gain curve, baseline, divides area perfect gain curve, baseline. meant represent amount potential gain \"captured\" model. gain_capture() metric identical accuracy ratio (AR), also sometimes called gini coefficient. two generally calculated cumulative accuracy profile curve, gain curve. See Engelmann reference information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Gain capture — gain_capture","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Gain capture — gain_capture","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Gain capture — gain_capture","text":"Engelmann, Bernd & Hayden, Evelyn & Tasche, Dirk (2003). \"Measuring Discriminative Power Rating Systems,\" Discussion Paper Series 2: Banking Financial Studies 2003,01, Deutsche Bundesbank.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Gain capture — gain_capture","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Gain capture — gain_capture","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. gain_capture(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture binary 0.879 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% gain_capture(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture macro 0.743 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% gain_capture(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture macro 0.743 # Groups are respected hpc_cv %>% group_by(Resample) %>% gain_capture(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 gain_capture macro 0.743 #> 2 Fold02 gain_capture macro 0.727 #> 3 Fold03 gain_capture macro 0.796 #> 4 Fold04 gain_capture macro 0.748 #> 5 Fold05 gain_capture macro 0.730 #> 6 Fold06 gain_capture macro 0.754 #> 7 Fold07 gain_capture macro 0.730 #> 8 Fold08 gain_capture macro 0.747 #> 9 Fold09 gain_capture macro 0.710 #> 10 Fold10 gain_capture macro 0.731 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% gain_capture(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 gain_capture macro_weighted 0.759 #> 2 Fold02 gain_capture macro_weighted 0.745 #> 3 Fold03 gain_capture macro_weighted 0.811 #> 4 Fold04 gain_capture macro_weighted 0.734 #> 5 Fold05 gain_capture macro_weighted 0.733 #> 6 Fold06 gain_capture macro_weighted 0.730 #> 7 Fold07 gain_capture macro_weighted 0.737 #> 8 Fold08 gain_capture macro_weighted 0.730 #> 9 Fold09 gain_capture macro_weighted 0.681 #> 10 Fold10 gain_capture macro_weighted 0.737 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") gain_capture_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.7428922 # --------------------------------------------------------------------------- # Visualize gain_capture() # Visually, this represents the area under the black curve, but above the # 45 degree line, divided by the area of the shaded triangle. library(ggplot2) autoplot(gain_curve(two_class_example, truth, Class1))"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Gain curve — gain_curve","title":"Gain curve — gain_curve","text":"gain_curve() constructs full gain curve returns tibble. See gain_capture() relevant area gain curve. Also see lift_curve() closely related concept.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Gain curve — gain_curve","text":"","code":"gain_curve(data, ...) # S3 method for data.frame gain_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Gain curve — gain_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Gain curve — gain_curve","text":"tibble class gain_df gain_grouped_df columns: .n index current sample. .n_events index current unique sample. Values repeated estimate values given identical indices column. .percent_tested cumulative percentage values tested. .percent_found cumulative percentage true results relative total number true results. using case_weights argument, columns weighted. makes sense frequency weights, integer weights representing number times particular observation repeated.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Gain curve — gain_curve","text":"ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples. greater area gain curve baseline, better model. Gain curves identical CAP curves (cumulative accuracy profile). See Engelmann reference information CAP curves.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"gain-and-lift-curves","dir":"Reference","previous_headings":"","what":"Gain and Lift Curves","title":"Gain curve — gain_curve","text":"motivation behind cumulative gain lift charts visual method determine effectiveness model compared results one might expect without model. example, without model, advertise random 10% customer base, might expect capture 10% total number positive responses advertised entire customer base. Given model predicts customers likely respond, hope can accurately target 10% customer base capture >10% total number positive responses. calculation construct gain curves follows: truth estimate placed descending order estimate values (estimate single column supplied ...). cumulative number samples true results relative entire number true results found. y-axis gain chart.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Gain curve — gain_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Gain curve — gain_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Gain curve — gain_curve","text":"Engelmann, Bernd & Hayden, Evelyn & Tasche, Dirk (2003). \"Measuring Discriminative Power Rating Systems,\" Discussion Paper Series 2: Banking Financial Studies 2003,01, Deutsche Bundesbank.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Gain curve — gain_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Gain curve — gain_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. gain_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 4 #> .n .n_events .percent_tested .percent_found #> #> 1 0 0 0 0 #> 2 1 1 0.2 0.388 #> 3 2 2 0.4 0.775 #> 4 3 3 0.6 1.16 #> 5 4 4 0.8 1.55 #> 6 5 5 1 1.94 #> 7 6 6 1.2 2.33 #> 8 7 7 1.4 2.71 #> 9 8 8 1.6 3.10 #> 10 9 9 1.8 3.49 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` library(ggplot2) library(dplyr) # Use autoplot to visualize # The top left hand corner of the grey triangle is a \"perfect\" gain curve autoplot(gain_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% gain_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples # The resample with the minimum (farthest to the left) \"perfect\" value is # used to draw the shaded region hpc_cv %>% group_by(Resample) %>% gain_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":null,"dir":"Reference","previous_headings":"","what":"Multiclass Probability Predictions — hpc_cv","title":"Multiclass Probability Predictions — hpc_cv","text":"Multiclass Probability Predictions","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Multiclass Probability Predictions — hpc_cv","text":"Kuhn, M., Johnson, K. (2013) Applied Predictive Modeling, Springer","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Multiclass Probability Predictions — hpc_cv","text":"hpc_cv data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Multiclass Probability Predictions — hpc_cv","text":"data frame contains predicted classes class probabilities linear discriminant analysis model fit HPC data set Kuhn Johnson (2013). data assessment sets 10-fold cross-validation scheme. data column columns true class (obs), class prediction (pred) columns class probability (columns VF, F, M, L). Additionally, column resample indicator included.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Multiclass Probability Predictions — hpc_cv","text":"","code":"data(hpc_cv) str(hpc_cv) #> 'data.frame':\t3467 obs. of 7 variables: #> $ obs : Factor w/ 4 levels \"VF\",\"F\",\"M\",\"L\": 1 1 1 1 1 1 1 1 1 1 ... #> $ pred : Factor w/ 4 levels \"VF\",\"F\",\"M\",\"L\": 1 1 1 1 1 1 1 1 1 1 ... #> $ VF : num 0.914 0.938 0.947 0.929 0.942 ... #> $ F : num 0.0779 0.0571 0.0495 0.0653 0.0543 ... #> $ M : num 0.00848 0.00482 0.00316 0.00579 0.00381 ... #> $ L : num 1.99e-05 1.01e-05 5.00e-06 1.56e-05 7.29e-06 ... #> $ Resample: chr \"Fold01\" \"Fold01\" \"Fold01\" \"Fold01\" ... # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section in any classification function (such as `?pr_auc`) to see how # to change this. levels(hpc_cv$obs) #> [1] \"VF\" \"F\" \"M\" \"L\""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Huber loss — huber_loss","title":"Huber loss — huber_loss","text":"Calculate Huber loss, loss function used robust regression. loss function less sensitive outliers rmse(). function quadratic small residual values linear large residual values.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Huber loss — huber_loss","text":"","code":"huber_loss(data, ...) # S3 method for data.frame huber_loss( data, truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... ) huber_loss_vec( truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Huber loss — huber_loss","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. delta single numeric value. Defines boundary loss function transitions quadratic linear. Defaults 1. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Huber loss — huber_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. huber_loss_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Huber loss — huber_loss","text":"Huber, P. (1964). Robust Estimation Location Parameter. Annals Statistics, 53 (1), 73-101.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Huber loss — huber_loss","text":"James Blair","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Huber loss — huber_loss","text":"","code":"# Supply truth and predictions as bare column names huber_loss(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 huber_loss standard 0.234 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% huber_loss(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 huber_loss standard 0.215 #> 2 10 huber_loss standard 0.212 #> 3 2 huber_loss standard 0.229 #> 4 3 huber_loss standard 0.197 #> 5 4 huber_loss standard 0.249 #> 6 5 huber_loss standard 0.208 #> 7 6 huber_loss standard 0.293 #> 8 7 huber_loss standard 0.268 #> 9 8 huber_loss standard 0.190 #> 10 9 huber_loss standard 0.218 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.228"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":null,"dir":"Reference","previous_headings":"","what":"Psuedo-Huber Loss — huber_loss_pseudo","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"Calculate Pseudo-Huber Loss, smooth approximation huber_loss(). Like huber_loss(), less sensitive outliers rmse().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"","code":"huber_loss_pseudo(data, ...) # S3 method for data.frame huber_loss_pseudo( data, truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... ) huber_loss_pseudo_vec( truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. delta single numeric value. Defines boundary loss function transitions quadratic linear. Defaults 1. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. huber_loss_pseudo_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"Huber, P. (1964). Robust Estimation Location Parameter. Annals Statistics, 53 (1), 73-101. Hartley, Richard (2004). Multiple View Geometry Computer Vision. (Second Edition). Page 619.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"James Blair","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"","code":"# Supply truth and predictions as bare column names huber_loss_pseudo(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 huber_loss_pseudo standard 0.199 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% huber_loss_pseudo(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 huber_loss_pseudo standard 0.185 #> 2 10 huber_loss_pseudo standard 0.179 #> 3 2 huber_loss_pseudo standard 0.196 #> 4 3 huber_loss_pseudo standard 0.168 #> 5 4 huber_loss_pseudo standard 0.212 #> 6 5 huber_loss_pseudo standard 0.177 #> 7 6 huber_loss_pseudo standard 0.246 #> 8 7 huber_loss_pseudo standard 0.227 #> 9 8 huber_loss_pseudo standard 0.161 #> 10 9 huber_loss_pseudo standard 0.188 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.194"},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":null,"dir":"Reference","previous_headings":"","what":"Index of ideality of correlation — iic","title":"Index of ideality of correlation — iic","text":"Calculate index ideality correlation. metric studied QSPR/QSAR models good criterion predictive potential models. highly dependent correlation coefficient well mean absolute error. Note application IIC useless two conditions: negative mean absolute error positive mean absolute error zero. outliers symmetric. Since outliers context dependent, please use checks validate whether restriction holds whether resulting IIC interpretative value. IIC seen alternative traditional correlation coefficient units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Index of ideality of correlation — iic","text":"","code":"iic(data, ...) # S3 method for data.frame iic(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) iic_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Index of ideality of correlation — iic","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Index of ideality of correlation — iic","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. iic_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Index of ideality of correlation — iic","text":"Toropova, . Toropov, . (2017). \"index ideality correlation. criterion predictability QSAR models skin permeability?\" Science Total Environment. 586: 466-472.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Index of ideality of correlation — iic","text":"Joyce Cahoon","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Index of ideality of correlation — iic","text":"","code":"# Supply truth and predictions as bare column names iic(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 iic standard 0.890 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% iic(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 iic standard 0.730 #> 2 10 iic standard 0.731 #> 3 2 iic standard 0.906 #> 4 3 iic standard 0.877 #> 5 4 iic standard 0.732 #> 6 5 iic standard 0.821 #> 7 6 iic standard 0.896 #> 8 7 iic standard 0.867 #> 9 8 iic standard 0.881 #> 10 9 iic standard 0.748 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.819"},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":null,"dir":"Reference","previous_headings":"","what":"J-index — j_index","title":"J-index — j_index","text":"Youden's J statistic defined : sens() + spec() - 1 related metric Informedness, see Details section relationship.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"J-index — j_index","text":"","code":"j_index(data, ...) # S3 method for data.frame j_index( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) j_index_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"J-index — j_index","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"J-index — j_index","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. j_index_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"J-index — j_index","text":"value J-index ranges [0, 1] 1 false positives false negatives. binary version J-index equivalent binary concept Informedness. Macro-weighted J-index equivalent multiclass informedness defined Powers, David M W (2011), equation (42).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"J-index — j_index","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"J-index — j_index","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"J-index — j_index","text":"Youden, W.J. (1950). \"Index rating diagnostic tests\". Cancer. 3: 32-35. Powers, David M W (2011). \"Evaluation: Precision, Recall F-Score ROC, Informedness, Markedness Correlation\". Journal Machine Learning Technologies. 2 (1): 37-63.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"J-index — j_index","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"J-index — j_index","text":"","code":"# Two class data(\"two_class_example\") j_index(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 j_index binary 0.673 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% j_index(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 j_index macro 0.434 # Groups are respected hpc_cv %>% group_by(Resample) %>% j_index(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 j_index macro 0.434 #> 2 Fold02 j_index macro 0.422 #> 3 Fold03 j_index macro 0.533 #> 4 Fold04 j_index macro 0.449 #> 5 Fold05 j_index macro 0.431 #> 6 Fold06 j_index macro 0.413 #> 7 Fold07 j_index macro 0.398 #> 8 Fold08 j_index macro 0.468 #> 9 Fold09 j_index macro 0.435 #> 10 Fold10 j_index macro 0.412 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% j_index(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 j_index macro_weighted 0.542 #> 2 Fold02 j_index macro_weighted 0.527 #> 3 Fold03 j_index macro_weighted 0.597 #> 4 Fold04 j_index macro_weighted 0.515 #> 5 Fold05 j_index macro_weighted 0.524 #> 6 Fold06 j_index macro_weighted 0.492 #> 7 Fold07 j_index macro_weighted 0.466 #> 8 Fold08 j_index macro_weighted 0.535 #> 9 Fold09 j_index macro_weighted 0.468 #> 10 Fold10 j_index macro_weighted 0.501 # Vector version j_index_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.6732334 # Making Class2 the \"relevant\" level j_index_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.6732334"},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":null,"dir":"Reference","previous_headings":"","what":"Kappa — kap","title":"Kappa — kap","text":"Kappa similar measure accuracy(), normalized accuracy expected chance alone useful one classes large frequency distributions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Kappa — kap","text":"","code":"kap(data, ...) # S3 method for data.frame kap( data, truth, estimate, weighting = \"none\", na_rm = TRUE, case_weights = NULL, ... ) kap_vec( truth, estimate, weighting = \"none\", na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Kappa — kap","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. weighting weighting apply computing scores. One : \"none\", \"linear\", \"quadratic\". Linear quadratic weighting penalizes mis-predictions \"far away\" true value. Note distance judged based ordering levels truth estimate. recommended provide ordered factors truth estimate explicitly code ordering, required. binary case, 3 weightings produce value, since ever possible 1 unit away true value. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Kappa — kap","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. kap_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Kappa — kap","text":"Kappa extends naturally multiclass scenarios. , macro micro averaging implemented.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Kappa — kap","text":"Cohen, J. (1960). \"coefficient agreement nominal scales\". Educational Psychological Measurement. 20 (1): 37-46. Cohen, J. (1968). \"Weighted kappa: Nominal scale agreement provision scaled disagreement partial credit\". Psychological Bulletin. 70 (4): 213-220.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Kappa — kap","text":"Max Kuhn Jon Harmon","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Kappa — kap","text":"","code":"library(dplyr) data(\"two_class_example\") data(\"hpc_cv\") # Two class kap(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 kap binary 0.675 # Multiclass # kap() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% kap(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 kap multiclass 0.533 # Groups are respected hpc_cv %>% group_by(Resample) %>% kap(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 kap multiclass 0.533 #> 2 Fold02 kap multiclass 0.512 #> 3 Fold03 kap multiclass 0.594 #> 4 Fold04 kap multiclass 0.511 #> 5 Fold05 kap multiclass 0.514 #> 6 Fold06 kap multiclass 0.486 #> 7 Fold07 kap multiclass 0.454 #> 8 Fold08 kap multiclass 0.531 #> 9 Fold09 kap multiclass 0.454 #> 10 Fold10 kap multiclass 0.492"},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Lift curve — lift_curve","title":"Lift curve — lift_curve","text":"lift_curve() constructs full lift curve returns tibble. See gain_curve() closely related concept.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Lift curve — lift_curve","text":"","code":"lift_curve(data, ...) # S3 method for data.frame lift_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Lift curve — lift_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Lift curve — lift_curve","text":"tibble class lift_df lift_grouped_df columns: .n index current sample. .n_events index current unique sample. Values repeated estimate values given identical indices column. .percent_tested cumulative percentage values tested. .lift First calculate cumulative percentage true results relative total number true results. divide .percent_tested. using case_weights argument, columns weighted. makes sense frequency weights, integer weights representing number times particular observation repeated.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Lift curve — lift_curve","text":"ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"gain-and-lift-curves","dir":"Reference","previous_headings":"","what":"Gain and Lift Curves","title":"Lift curve — lift_curve","text":"motivation behind cumulative gain lift charts visual method determine effectiveness model compared results one might expect without model. example, without model, advertise random 10% customer base, might expect capture 10% total number positive responses advertised entire customer base. Given model predicts customers likely respond, hope can accurately target 10% customer base capture >10% total number positive responses. calculation construct lift curves follows: truth estimate placed descending order estimate values (estimate single column supplied ...). cumulative number samples true results relative entire number true results found. cumulative % found divided cumulative % tested construct lift value. ratio represents factor improvement uninformed model. Values >1 represent valuable model. y-axis lift chart.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Lift curve — lift_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Lift curve — lift_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Lift curve — lift_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Lift curve — lift_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. lift_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 4 #> .n .n_events .percent_tested .lift #> #> 1 0 0 0 NaN #> 2 1 1 0.2 1.94 #> 3 2 2 0.4 1.94 #> 4 3 3 0.6 1.94 #> 5 4 4 0.8 1.94 #> 6 5 5 1 1.94 #> 7 6 6 1.2 1.94 #> 8 7 7 1.4 1.94 #> 9 8 8 1.6 1.94 #> 10 9 9 1.8 1.94 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` library(ggplot2) library(dplyr) # Use autoplot to visualize autoplot(lift_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% lift_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% lift_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":null,"dir":"Reference","previous_headings":"","what":"Survival Analysis Results — lung_surv","title":"Survival Analysis Results — lung_surv","text":"Survival Analysis Results","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Survival Analysis Results — lung_surv","text":"lung_surv data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Survival Analysis Results — lung_surv","text":"data contain plausible results applying predictive survival models lung data set using censored package.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Survival Analysis Results — lung_surv","text":"","code":"data(lung_surv) str(lung_surv) #> tibble [228 × 3] (S3: tbl_df/tbl/data.frame) #> $ .pred :List of 228 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.819 0.597 0.407 0.264 0.164 #> .. ..$ .weight_time : num [1:5] 100 200 300 306 306 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 455 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.627 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.59 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.763 0.625 0.498 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 200 210 210 210 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.915 0.915 0.915 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.09 1.09 1.09 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.613 0.484 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.819 0.597 0.407 0.264 0.164 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.841 0.64 0.46 0.316 0.209 #> .. ..$ .weight_time : num [1:5] 100 200 300 310 310 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.837 0.631 0.449 0.305 0.2 #> .. ..$ .weight_time : num [1:5] 100 200 300 361 361 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.707 0.707 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.41 1.41 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.851 0.659 0.484 0.341 0.233 #> .. ..$ .weight_time : num [1:5] 100 200 218 218 218 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.909 0.909 0.909 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.1 1.1 1.1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7568 0.4871 0.2858 0.1563 0.0806 #> .. ..$ .weight_time : num [1:5] 100 166 166 166 166 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 170 170 170 170 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.841 0.64 0.46 0.316 0.209 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.614 0.485 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] NA NA NA NA NA #> .. ..$ .weight_time : num [1:5] 71 71 71 71 71 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 144 144 144 144 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7524 0.4799 0.2785 0.1504 0.0765 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.858 0.674 0.503 0.361 0.251 #> .. ..$ .weight_time : num [1:5] 61 61 61 61 61 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 88 88 88 88 88 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 200 300 301 301 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.752 0.752 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.33 1.33 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.945 0.865 0.777 0.688 0.602 #> .. ..$ .weight_time : num [1:5] 81 81 81 81 81 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.855 0.668 0.495 0.353 0.243 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.759 0.619 0.491 0.381 #> .. ..$ .weight_time : num [1:5] 100 200 300 371 371 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.697 0.697 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.44 1.44 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.884 0.728 0.575 0.44 0.329 #> .. ..$ .weight_time : num [1:5] 100 200 300 394 394 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.663 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.51 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.895 0.751 0.607 0.478 0.367 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.613 0.484 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.6149 0.2851 0.1125 0.0392 0.0123 #> .. ..$ .weight_time : num [1:5] 100 118 118 118 118 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.851 0.659 0.484 0.341 0.233 #> .. ..$ .weight_time : num [1:5] 100 200 300 390 390 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.663 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.51 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 12 12 12 12 12 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.896 0.753 0.611 0.481 0.371 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 473 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.73 0.443 0.243 0.123 0.058 #> .. ..$ .weight_time : num [1:5] 26 26 26 26 26 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.783 0.533 0.334 0.197 0.11 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.853 0.663 0.489 0.346 0.237 #> .. ..$ .weight_time : num [1:5] 100 107 107 107 107 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7568 0.4871 0.2858 0.1563 0.0806 #> .. ..$ .weight_time : num [1:5] 53 53 53 53 53 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.85 0.657 0.481 0.338 0.23 #> .. ..$ .weight_time : num [1:5] 100 122 122 122 122 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.748 0.4727 0.2713 0.1446 0.0725 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.76 0.62 0.492 0.382 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 93 93 93 93 93 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.764 0.626 0.499 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 460 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.834 0.625 0.441 0.297 0.193 #> .. ..$ .weight_time : num [1:5] 100 153 153 153 153 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.94 0.851 0.756 0.66 0.569 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 433 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.853 0.663 0.489 0.346 0.237 #> .. ..$ .weight_time : num [1:5] 100 145 145 145 145 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.829 0.615 0.429 0.286 0.183 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.829 0.616 0.43 0.286 0.183 #> .. ..$ .weight_time : num [1:5] 95 95 95 95 95 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.882 0.723 0.568 0.433 0.321 #> .. ..$ .weight_time : num [1:5] 100 200 300 303 303 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.744 0.744 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.34 1.34 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.836 0.63 0.448 0.304 0.199 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.882 0.723 0.568 0.433 0.321 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.914 0.792 0.666 0.548 0.442 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.893 0.747 0.601 0.47 0.359 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 189 189 189 189 #> .. ..$ .pred_censored : num [1:5] 0.995 0.954 0.954 0.954 0.954 #> .. ..$ .weight_censored: num [1:5] 1.01 1.05 1.05 1.05 1.05 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 53 53 53 53 53 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.759 0.619 0.491 0.381 #> .. ..$ .weight_time : num [1:5] 100 200 246 246 246 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.845 0.845 0.845 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.18 1.18 1.18 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.842 0.642 0.463 0.319 0.212 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.895 0.751 0.607 0.477 0.366 #> .. ..$ .weight_time : num [1:5] 65 65 65 65 65 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.936 0.843 0.743 0.643 0.55 #> .. ..$ .weight_time : num [1:5] 5 5 5 5 5 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7653 0.5014 0.3005 0.1683 0.0891 #> .. ..$ .weight_time : num [1:5] 100 132 132 132 132 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.907 0.776 0.643 0.52 0.412 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.764 0.626 0.499 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 345 345 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.717 0.717 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.39 1.39 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.619 0.434 0.29 0.187 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 444 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.858 0.673 0.502 0.36 0.25 #> .. ..$ .weight_time : num [1:5] 100 200 223 223 223 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.895 0.895 0.895 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.12 1.12 1.12 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.82 0.6 0.411 0.267 0.167 #> .. ..$ .weight_time : num [1:5] 100 175 175 175 175 #> .. ..$ .pred_censored : num [1:5] 0.995 0.978 0.978 0.978 0.978 #> .. ..$ .weight_censored: num [1:5] 1.01 1.02 1.02 1.02 1.02 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.9 0.762 0.623 0.496 0.386 #> .. ..$ .weight_time : num [1:5] 60 60 60 60 60 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.827 0.612 0.426 0.282 0.179 #> .. ..$ .weight_time : num [1:5] 100 163 163 163 163 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7413 0.4618 0.2605 0.1362 0.0668 #> .. ..$ .weight_time : num [1:5] 65 65 65 65 65 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.843 0.643 0.463 0.32 0.213 #> .. ..$ .weight_time : num [1:5] 100 200 208 208 208 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.915 0.915 0.915 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.09 1.09 1.09 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.937 0.844 0.745 0.646 0.553 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 428 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 200 230 230 230 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.874 0.874 0.874 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.14 1.14 1.14 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.915 0.796 0.672 0.554 0.449 #> .. ..$ .weight_time : num [1:5] 100 200 300 305 305 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 11 11 11 11 11 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.868 0.694 0.53 0.39 0.279 #> .. ..$ .weight_time : num [1:5] 100 132 132 132 132 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.911 0.786 0.658 0.537 0.431 #> .. ..$ .weight_time : num [1:5] 100 200 226 226 226 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.874 0.874 0.874 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.14 1.14 1.14 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.749 0.604 0.474 0.363 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 426 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.944 0.862 0.773 0.682 0.595 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.908 0.78 0.649 0.527 0.419 #> .. ..$ .weight_time : num [1:5] 100 200 300 363 363 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.707 0.707 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.41 1.41 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.874 0.706 0.545 0.407 0.295 #> .. ..$ .weight_time : num [1:5] 11 11 11 11 11 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.883 0.725 0.572 0.437 0.325 #> .. ..$ .weight_time : num [1:5] 100 176 176 176 176 #> .. ..$ .pred_censored : num [1:5] 0.995 0.972 0.972 0.972 0.972 #> .. ..$ .weight_censored: num [1:5] 1.01 1.03 1.03 1.03 1.03 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.898 0.757 0.616 0.488 0.377 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.848 0.654 0.477 0.334 0.226 #> .. ..$ .weight_time : num [1:5] 95 95 95 95 95 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.866 0.689 0.523 0.383 0.272 #> .. ..$ .weight_time : num [1:5] 100 NA NA NA NA #> .. ..$ .pred_censored : num [1:5] 0.995 NA NA NA NA #> .. ..$ .weight_censored: num [1:5] 1.01 NA NA NA NA #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.918 0.803 0.682 0.568 0.464 #> .. ..$ .weight_time : num [1:5] 100 167 167 167 167 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.863 0.684 0.516 0.375 0.264 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.824 0.606 0.418 0.275 0.173 #> .. ..$ .weight_time : num [1:5] 100 200 284 284 284 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.793 0.793 0.793 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.26 1.26 1.26 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.903 0.768 0.632 0.506 0.397 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.896 0.753 0.61 0.48 0.37 #> .. ..$ .weight_time : num [1:5] 100 147 147 147 147 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.918 0.803 0.682 0.568 0.464 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7321 0.4471 0.2462 0.1253 0.0597 #> .. ..$ .weight_time : num [1:5] 100 163 163 163 163 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 239 239 239 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.86 0.86 0.86 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.16 1.16 1.16 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.832 0.621 0.437 0.293 0.189 #> .. ..$ .weight_time : num [1:5] 88 88 88 88 88 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.907 0.778 0.646 0.524 0.416 #> .. ..$ .weight_time : num [1:5] 100 200 245 245 245 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.845 0.845 0.845 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.18 1.18 1.18 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.933 0.837 0.733 0.632 0.536 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7321 0.4471 0.2462 0.1253 0.0597 #> .. ..$ .weight_time : num [1:5] 30 30 30 30 30 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.827 0.612 0.426 0.282 0.179 #> .. ..$ .weight_time : num [1:5] 100 179 179 179 179 #> .. ..$ .pred_censored : num [1:5] 0.995 0.966 0.966 0.966 0.966 #> .. ..$ .weight_censored: num [1:5] 1.01 1.03 1.03 1.03 1.03 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.824 0.606 0.418 0.275 0.173 #> .. ..$ .weight_time : num [1:5] 100 200 300 310 310 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.835 0.627 0.444 0.3 0.196 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 477 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> .. [list output truncated] #> $ .pred_time: num [1:228] 324 476 521 368 506 ... #> $ surv_obj : 'Surv' num [1:228, 1:2] 306 455 1010+ 210 883 1022+ 310 361 218 166 ... #> ..- attr(*, \"dimnames\")=List of 2 #> .. ..$ : NULL #> .. ..$ : chr [1:2] \"time\" \"status\" #> ..- attr(*, \"type\")= chr \"right\" # `surv_obj` is a `Surv()` object"},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute error — mae","title":"Mean absolute error — mae","text":"Calculate mean absolute error. metric units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute error — mae","text":"","code":"mae(data, ...) # S3 method for data.frame mae(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mae_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute error — mae","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute error — mae","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mae_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute error — mae","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute error — mae","text":"","code":"# Supply truth and predictions as bare column names mae(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mae standard 0.545 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mae(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mae standard 0.512 #> 2 10 mae standard 0.504 #> 3 2 mae standard 0.545 #> 4 3 mae standard 0.496 #> 5 4 mae standard 0.587 #> 6 5 mae standard 0.500 #> 7 6 mae standard 0.627 #> 8 7 mae standard 0.566 #> 9 8 mae standard 0.473 #> 10 9 mae standard 0.540 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.535"},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute percent error — mape","title":"Mean absolute percent error — mape","text":"Calculate mean absolute percentage error. metric relative units.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute percent error — mape","text":"","code":"mape(data, ...) # S3 method for data.frame mape(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mape_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute percent error — mape","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute percent error — mape","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mape_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean absolute percent error — mape","text":"Note value Inf returned mape() observed value negative.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute percent error — mape","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute percent error — mape","text":"","code":"# Supply truth and predictions as bare column names mape(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mape standard Inf library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mape(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mape standard Inf #> 2 10 mape standard Inf #> 3 2 mape standard Inf #> 4 3 mape standard 28.5 #> 5 4 mape standard 95.3 #> 6 5 mape standard Inf #> 7 6 mape standard 73.1 #> 8 7 mape standard Inf #> 9 8 mape standard Inf #> 10 9 mape standard 37.9 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 Inf"},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute scaled error — mase","title":"Mean absolute scaled error — mase","text":"Calculate mean absolute scaled error. metric scale independent symmetric. generally used comparing forecast error time series settings. Due time series nature metric, necessary order observations ascending order time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute scaled error — mase","text":"","code":"mase(data, ...) # S3 method for data.frame mase( data, truth, estimate, m = 1L, mae_train = NULL, na_rm = TRUE, case_weights = NULL, ... ) mase_vec( truth, estimate, m = 1L, mae_train = NULL, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute scaled error — mase","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. m integer value number lags used calculate -sample seasonal naive error. default used non-seasonal time series. observation daily level data showed weekly seasonality, m = 7L reasonable choice 7-day seasonal naive calculation. mae_train numeric value allows user provide -sample seasonal naive mean absolute error. value provided, --sample seasonal naive mean absolute error calculated truth used instead. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute scaled error — mase","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mase_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean absolute scaled error — mase","text":"mase() different numeric metrics. original implementation mase() calls using -sample naive mean absolute error compute scaled errors . uses instead --sample error chance --sample error computed forecasting short horizon (.e. sample size 1 2). However, yardstick knows --sample truth estimate values. , --sample error used computation default. -sample naive mean absolute error required known, can passed mae_train argument used instead. -sample data available, naive mean absolute error can easily computed mae(data, truth, lagged_truth).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Mean absolute scaled error — mase","text":"Rob J. Hyndman (2006). ANOTHER LOOK FORECAST-ACCURACY METRICS INTERMITTENT DEMAND. Foresight, 4, 46.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute scaled error — mase","text":"Alex Hallam","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute scaled error — mase","text":"","code":"# Supply truth and predictions as bare column names mase(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mase(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mase standard 0.256 #> 2 10 mase standard 0.240 #> 3 2 mase standard 0.238 #> 4 3 mase standard 0.219 #> 5 4 mase standard 0.229 #> 6 5 mase standard 0.261 #> 7 6 mase standard 0.217 #> 8 7 mase standard 0.267 #> 9 8 mase standard 0.216 #> 10 9 mase standard 0.251 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.240"},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":null,"dir":"Reference","previous_headings":"","what":"Matthews correlation coefficient — mcc","title":"Matthews correlation coefficient — mcc","text":"Matthews correlation coefficient","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Matthews correlation coefficient — mcc","text":"","code":"mcc(data, ...) # S3 method for data.frame mcc(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mcc_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Matthews correlation coefficient — mcc","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Matthews correlation coefficient — mcc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mcc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Matthews correlation coefficient — mcc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Matthews correlation coefficient — mcc","text":"mcc() known multiclass generalization computed automatically factor 2 levels provided. , averaging methods provided.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Matthews correlation coefficient — mcc","text":"Giuseppe, J. (2012). \"Comparison MCC CEN Error Measures Multi-Class Prediction\". PLOS ONE. Vol 7, Iss 8, e41882.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Matthews correlation coefficient — mcc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Matthews correlation coefficient — mcc","text":"","code":"library(dplyr) data(\"two_class_example\") data(\"hpc_cv\") # Two class mcc(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mcc binary 0.677 # Multiclass # mcc() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% mcc(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mcc multiclass 0.542 # Groups are respected hpc_cv %>% group_by(Resample) %>% mcc(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 mcc multiclass 0.542 #> 2 Fold02 mcc multiclass 0.521 #> 3 Fold03 mcc multiclass 0.602 #> 4 Fold04 mcc multiclass 0.519 #> 5 Fold05 mcc multiclass 0.520 #> 6 Fold06 mcc multiclass 0.494 #> 7 Fold07 mcc multiclass 0.461 #> 8 Fold08 mcc multiclass 0.538 #> 9 Fold09 mcc multiclass 0.459 #> 10 Fold10 mcc multiclass 0.498"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for summarizing new metrics — metric-summarizers","title":"Developer function for summarizing new metrics — metric-summarizers","text":"numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), static_survival_metric_summarizer() useful alongside check_metric yardstick_remove_missing implementing new custom metrics. functions call metric function inside dplyr::summarise() dplyr::reframe() curve_metric_summarizer(). See Custom performance metrics information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for summarizing new metrics — metric-summarizers","text":"","code":"numeric_metric_summarizer( name, fn, data, truth, estimate, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) class_metric_summarizer( name, fn, data, truth, estimate, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) prob_metric_summarizer( name, fn, data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) curve_metric_summarizer( name, fn, data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) dynamic_survival_metric_summarizer( name, fn, data, truth, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) static_survival_metric_summarizer( name, fn, data, truth, estimate, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) curve_survival_metric_summarizer( name, fn, data, truth, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for summarizing new metrics — metric-summarizers","text":"name single character representing name metric use tibble output. modified include type averaging appropriate. fn vector version custom metric function. generally takes truth, estimate, na_rm, extra arguments needed calculate metric. data data frame truth estimate columns passed data frame version metric function called numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), static_survival_metric_summarizer(). truth unquoted column name corresponding truth column. estimate Generally, unquoted column name corresponding estimate column. metrics take multiple columns ... like class probability metrics, result dots_to_estimate(). ... dots future extensions must empty. na_rm logical value indicating whether NA values stripped computation proceeds. removal executed yardstick_remove_missing(). case_weights metrics supporting case weights, unquoted column name corresponding case weights can passed . NULL, case weights passed fn named argument case_weights. fn_options named list metric specific options. spliced metric function call using !!! rlang. default results nothing spliced call. error_call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. estimator can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use. event_level can either NULL use default event_level value fn single string either \"first\" \"second\" pass along describing level considered \"event\".","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Developer function for summarizing new metrics — metric-summarizers","text":"numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), dynamic_survival_metric_summarizer() generally called data frame version metric function. knows call metric grouped data frames returns tibble consistent metrics.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":null,"dir":"Reference","previous_headings":"","what":"Combine metric functions — metric_set","title":"Combine metric functions — metric_set","text":"metric_set() allows combine multiple metric functions together new function calculates .","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Combine metric functions — metric_set","text":"","code":"metric_set(...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Combine metric functions — metric_set","text":"... bare names functions included metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Combine metric functions — metric_set","text":"functions must either: numeric metrics mix class metrics class prob metrics mix dynamic, integrated, static survival metrics instance, rmse() can used mae() numeric metrics, accuracy() classification metric. accuracy() can used roc_auc(). returned metric function different argument list depending whether numeric metrics mix class/prob metrics passed . mixing class class prob metrics, pass hard predictions (factor column) named argument estimate, soft predictions (class probability columns) bare column names tidyselect selectors .... mixing dynamic, integrated, static survival metrics, pass time predictions named argument estimate, survival predictions bare column names tidyselect selectors .... metric_tweak() used \"tweak\" one arguments, like estimator event_level, tweaked version wins. allows set estimator metric metric basis still use metric_set().","code":"# Numeric metric set signature: fn( data, truth, estimate, na_rm = TRUE, case_weights = NULL, ... ) # Class / prob metric set signature: fn( data, truth, ..., estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) # Dynamic / integrated / static survival metric set signature: fn( data, truth, ..., estimate, na_rm = TRUE, case_weights = NULL )"},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Combine metric functions — metric_set","text":"","code":"library(dplyr) # Multiple regression metrics multi_metric <- metric_set(rmse, rsq, ccc) # The returned function has arguments: # fn(data, truth, estimate, na_rm = TRUE, ...) multi_metric(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 ccc standard 0.937 # Groups are respected on the new metric function class_metrics <- metric_set(accuracy, kap) hpc_cv %>% group_by(Resample) %>% class_metrics(obs, estimate = pred) #> # A tibble: 20 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492 # --------------------------------------------------------------------------- # If you need to set options for certain metrics, # do so by wrapping the metric and setting the options inside the wrapper, # passing along truth and estimate as quoted arguments. # Then add on the function class of the underlying wrapped function, # and the direction of optimization. ccc_with_bias <- function(data, truth, estimate, na_rm = TRUE, ...) { ccc( data = data, truth = !!rlang::enquo(truth), estimate = !!rlang::enquo(estimate), # set bias = TRUE bias = TRUE, na_rm = na_rm, ... ) } # Use `new_numeric_metric()` to formalize this new metric function ccc_with_bias <- new_numeric_metric(ccc_with_bias, \"maximize\") multi_metric2 <- metric_set(rmse, rsq, ccc_with_bias) multi_metric2(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 ccc standard 0.937 # --------------------------------------------------------------------------- # A class probability example: # Note that, when given class or class prob functions, # metric_set() returns a function with signature: # fn(data, truth, ..., estimate) # to be able to mix class and class prob metrics. # You must provide the `estimate` column by explicitly naming # the argument class_and_probs_metrics <- metric_set(roc_auc, pr_auc, accuracy) hpc_cv %>% group_by(Resample) %>% class_and_probs_metrics(obs, VF:L, estimate = pred) #> # A tibble: 30 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> # ℹ 20 more rows"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for summarizing new metrics — metric_summarizer","title":"Developer function for summarizing new metrics — metric_summarizer","text":"metric_summarizer() soft-deprecated yardstick 1.2.0. Please switch use class_metric_summarizer(), numeric_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for summarizing new metrics — metric_summarizer","text":"","code":"metric_summarizer( metric_nm, metric_fn, data, truth, estimate, estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, ..., metric_fn_options = list() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for summarizing new metrics — metric_summarizer","text":"metric_nm single character representing name metric use tibble output. modified include type averaging appropriate. metric_fn vector version custom metric function. generally takes truth, estimate, na_rm, extra arguments needed calculate metric. data data frame truth estimate columns passed data frame version metric function called metric_summarizer(). truth unquoted column name corresponding truth column. estimate Generally, unquoted column name corresponding estimate column. metrics take multiple columns ... like class probability metrics, result dots_to_estimate(). estimator numeric metrics, left NULL averaging passed metric function implementation. classification metrics, can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use. na_rm logical value indicating whether NA values stripped computation proceeds. removal executed metric_vec_template(). event_level numeric metrics, left NULL prevent passed metric function implementation. classification metrics, can either NULL use default event_level value metric_fn single string either \"first\" \"second\" pass along describing level considered \"event\". case_weights metrics supporting case weights, unquoted column name corresponding case weights can passed . NULL, case weights passed metric_fn named argument case_weights. ... Currently used. Metric specific options passed metric_fn_options. metric_fn_options named list metric specific options. spliced metric function call using !!! rlang. default results nothing spliced call.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":null,"dir":"Reference","previous_headings":"","what":"Tweak a metric function — metric_tweak","title":"Tweak a metric function — metric_tweak","text":"metric_tweak() allows tweak existing metric .fn, giving new .name setting new optional argument defaults .... similar purrr::partial(), designed specifically yardstick metrics. metric_tweak() especially useful constructing metric_set() tuning tune package. metric set constructed, way adjust value optional arguments (beta f_meas()). Using metric_tweak(), can set optional arguments custom values ahead time, go metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Tweak a metric function — metric_tweak","text":"","code":"metric_tweak(.name, .fn, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Tweak a metric function — metric_tweak","text":".name single string giving name new metric. used \".metric\" column output. .fn existing yardstick metric function tweak. ... Name-value pairs specifying optional arguments override values replace . Arguments data, truth, estimate considered protected, overridden, optional arguments can altered.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Tweak a metric function — metric_tweak","text":"tweaked version .fn, updated use new defaults supplied ....","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Tweak a metric function — metric_tweak","text":"function returned metric_tweak() takes ... arguments, passed original .fn. Passing data, truth, estimate position generally safe, recommended pass optional arguments name ensure evaluated correctly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Tweak a metric function — metric_tweak","text":"","code":"mase12 <- metric_tweak(\"mase12\", mase, m = 12) # Defaults to `m = 1` mase(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 # Updated to use `m = 12`. `mase12()` has this set already. mase(solubility_test, solubility, prediction, m = 12) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 0.582 mase12(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase12 standard 0.582 # This is most useful to set optional argument values ahead of time when # using a metric set mase10 <- metric_tweak(\"mase10\", mase, m = 10) metrics <- metric_set(mase, mase10, mase12) metrics(solubility_test, solubility, prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 #> 2 mase10 standard 0.664 #> 3 mase12 standard 0.582"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for calling new metrics — metric_vec_template","title":"Developer function for calling new metrics — metric_vec_template","text":"metric_vec_template() soft-deprecated yardstick 1.2.0. Please switch use check_metric yardstick_remove_missing functions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for calling new metrics — metric_vec_template","text":"","code":"metric_vec_template( metric_impl, truth, estimate, na_rm = TRUE, cls = \"numeric\", estimator = NULL, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for calling new metrics — metric_vec_template","text":"metric_impl core implementation function custom metric. core implementation function generally defined inside vector method metric function. truth realized vector truth. either factor numeric. estimate realized estimate result. either numeric vector, factor vector, numeric matrix (case multiple class probability columns) depending metric function. na_rm logical value indicating whether NA values stripped computation proceeds. NA values removed getting core implementation function worry handling . na_rm=FALSE NA values exist, NA automatically returned. cls character vector length 1 2 corresponding class truth estimate , respectively. truth estimate class, just supply vector length 1. different, supply vector length 2. matrices, best supply \"numeric\" class check . estimator type averaging use. point, averaging type finalized, character vector length 1\\. default, character value required one : \"binary\", \"macro\", \"micro\", \"macro_weighted\". metric allows less averaging methods, override averaging_override. case_weights Optionally, realized case weights, numeric vector. must length truth, considered na_rm checks. supplied, passed metric_impl named argument case_weights. ... Extra arguments core metric function, metric_impl, can technically passed , generally extra args added R's scoping rules core metric function created fly vector method called.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"General Function to Estimate Performance — metrics","title":"General Function to Estimate Performance — metrics","text":"function estimates one common performance estimates depending class truth (see Value ) returns three column tibble.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"General Function to Estimate Performance — metrics","text":"","code":"metrics(data, ...) # S3 method for data.frame metrics(data, truth, estimate, ..., na_rm = TRUE, options = list())"},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General Function to Estimate Performance — metrics","text":"data data.frame containing columns specified truth, estimate, .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true results (numeric factor). unquoted column name although argument passed expression support quasiquotation (can unquote column names). estimate column identifier predicted results (also numeric factor). truth can specified different ways primary method use unquoted variable name. na_rm logical value indicating whether NA values stripped computation proceeds. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"General Function to Estimate Performance — metrics","text":"three column tibble. truth factor, rows accuracy() Kappa statistic (kap()). truth two levels 1 column class probabilities passed ..., rows two class versions mn_log_loss() roc_auc(). truth two levels full set class probabilities passed ..., rows multiclass version mn_log_loss() Hand Till generalization roc_auc(). truth numeric, rows rmse(), rsq(), mae().","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"General Function to Estimate Performance — metrics","text":"","code":"# Accuracy and kappa metrics(two_class_example, truth, predicted) #> # A tibble: 2 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 # Add on multinomal log loss and ROC AUC by specifying class prob columns metrics(two_class_example, truth, predicted, Class1) #> # A tibble: 4 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 mn_log_loss binary 0.328 #> 4 roc_auc binary 0.939 # Regression metrics metrics(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 mae standard 0.545 # Multiclass metrics work, but you cannot specify any averaging # for roc_auc() besides the default, hand_till. Use the specific function # if you need more customization library(dplyr) hpc_cv %>% group_by(Resample) %>% metrics(obs, pred, VF:L) %>% print(n = 40) #> # A tibble: 40 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492 #> 21 Fold01 mn_log_loss multiclass 0.734 #> 22 Fold02 mn_log_loss multiclass 0.808 #> 23 Fold03 mn_log_loss multiclass 0.705 #> 24 Fold04 mn_log_loss multiclass 0.747 #> 25 Fold05 mn_log_loss multiclass 0.799 #> 26 Fold06 mn_log_loss multiclass 0.766 #> 27 Fold07 mn_log_loss multiclass 0.927 #> 28 Fold08 mn_log_loss multiclass 0.855 #> 29 Fold09 mn_log_loss multiclass 0.861 #> 30 Fold10 mn_log_loss multiclass 0.821 #> 31 Fold01 roc_auc hand_till 0.813 #> 32 Fold02 roc_auc hand_till 0.817 #> 33 Fold03 roc_auc hand_till 0.869 #> 34 Fold04 roc_auc hand_till 0.849 #> 35 Fold05 roc_auc hand_till 0.811 #> 36 Fold06 roc_auc hand_till 0.836 #> 37 Fold07 roc_auc hand_till 0.825 #> 38 Fold08 roc_auc hand_till 0.846 #> 39 Fold09 roc_auc hand_till 0.828 #> 40 Fold10 roc_auc hand_till 0.812"},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean log loss for multinomial data — mn_log_loss","title":"Mean log loss for multinomial data — mn_log_loss","text":"Compute logarithmic loss classification model.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean log loss for multinomial data — mn_log_loss","text":"","code":"mn_log_loss(data, ...) # S3 method for data.frame mn_log_loss( data, truth, ..., na_rm = TRUE, sum = FALSE, event_level = yardstick_event_level(), case_weights = NULL ) mn_log_loss_vec( truth, estimate, na_rm = TRUE, sum = FALSE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean log loss for multinomial data — mn_log_loss","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. sum logical. sum likelihood contributions returned (instead mean value)? event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean log loss for multinomial data — mn_log_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mn_log_loss_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean log loss for multinomial data — mn_log_loss","text":"Log loss measure performance classification model. perfect model log loss 0. Compared accuracy(), log loss takes account uncertainty prediction gives detailed view actual performance. example, given two input probabilities .6 .9 classified predicting positive value, say, \"Yes\", accuracy metric interpret value. true output \"Yes\", log loss penalizes .6 \"less sure\" result compared probability .9.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Mean log loss for multinomial data — mn_log_loss","text":"Log loss known multiclass extension, simply sum log loss values class prediction. , averaging types supported.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean log loss for multinomial data — mn_log_loss","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean log loss for multinomial data — mn_log_loss","text":"","code":"# Two class data(\"two_class_example\") mn_log_loss(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328 # Multiclass library(dplyr) data(hpc_cv) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% mn_log_loss(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss multiclass 0.734 # Groups are respected hpc_cv %>% group_by(Resample) %>% mn_log_loss(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 mn_log_loss multiclass 0.734 #> 2 Fold02 mn_log_loss multiclass 0.808 #> 3 Fold03 mn_log_loss multiclass 0.705 #> 4 Fold04 mn_log_loss multiclass 0.747 #> 5 Fold05 mn_log_loss multiclass 0.799 #> 6 Fold06 mn_log_loss multiclass 0.766 #> 7 Fold07 mn_log_loss multiclass 0.927 #> 8 Fold08 mn_log_loss multiclass 0.855 #> 9 Fold09 mn_log_loss multiclass 0.861 #> 10 Fold10 mn_log_loss multiclass 0.821 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") mn_log_loss_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.7338423 # Supply `...` with quasiquotation prob_cols <- levels(two_class_example$truth) mn_log_loss(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328 mn_log_loss(two_class_example, truth, !!prob_cols[1]) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328"},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean percentage error — mpe","title":"Mean percentage error — mpe","text":"Calculate mean percentage error. metric relative units. can used measure estimate's bias. Note truth values 0, value : -Inf (estimate > 0), Inf (estimate < 0), NaN (estimate == 0) returned mpe().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean percentage error — mpe","text":"","code":"mpe(data, ...) # S3 method for data.frame mpe(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mpe_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean percentage error — mpe","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean percentage error — mpe","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mpe_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean percentage error — mpe","text":"Thomas Bierhance","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean percentage error — mpe","text":"","code":"# `solubility_test$solubility` has zero values with corresponding # `$prediction` values that are negative. By definition, this causes `Inf` # to be returned from `mpe()`. solubility_test[solubility_test$solubility == 0, ] #> solubility prediction #> 17 0 -0.1532030 #> 220 0 -0.3876578 mpe(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mpe standard Inf # We'll remove the zero values for demonstration solubility_test <- solubility_test[solubility_test$solubility != 0, ] # Supply truth and predictions as bare column names mpe(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mpe standard 16.1 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mpe(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mpe standard -56.2 #> 2 10 mpe standard 50.4 #> 3 2 mpe standard -27.9 #> 4 3 mpe standard 0.470 #> 5 4 mpe standard -0.836 #> 6 5 mpe standard -35.3 #> 7 6 mpe standard 7.51 #> 8 7 mpe standard -34.5 #> 9 8 mpe standard 7.87 #> 10 9 mpe standard 14.7 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 -7.38"},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean signed deviation — msd","title":"Mean signed deviation — msd","text":"Mean signed deviation (also known mean signed difference, mean signed error) computes average differences truth estimate. related metric mean absolute error (mae()).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean signed deviation — msd","text":"","code":"msd(data, ...) # S3 method for data.frame msd(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) msd_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean signed deviation — msd","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean signed deviation — msd","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. msd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean signed deviation — msd","text":"Mean signed deviation rarely used, since positive negative errors cancel . example, msd_vec(c(100, -100), c(0, 0)) return seemingly \"perfect\" value 0, even though estimate wildly different truth. mae() attempts remedy taking absolute value differences computing mean. metric computed mean(truth - estimate), following convention \"error\" computed observed - predicted. expected metric computed mean(estimate - truth), reverse sign result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean signed deviation — msd","text":"Thomas Bierhance","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean signed deviation — msd","text":"","code":"# Supply truth and predictions as bare column names msd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 msd standard -0.0143 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% msd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 msd standard -0.0119 #> 2 10 msd standard -0.0424 #> 3 2 msd standard 0.0111 #> 4 3 msd standard -0.0906 #> 5 4 msd standard -0.0859 #> 6 5 msd standard -0.0301 #> 7 6 msd standard -0.0132 #> 8 7 msd standard -0.00640 #> 9 8 msd standard -0.000697 #> 10 9 msd standard -0.0399 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 -0.0310"},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct a new metric function — new-metric","title":"Construct a new metric function — new-metric","text":"functions provide convenient wrappers create three types metric functions yardstick: numeric metrics, class metrics, class probability metrics. add metric-specific class fn attach direction attribute. features used metric_set() tune model tuning. See Custom performance metrics information creating custom metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct a new metric function — new-metric","text":"","code":"new_class_metric(fn, direction) new_prob_metric(fn, direction) new_numeric_metric(fn, direction) new_dynamic_survival_metric(fn, direction) new_integrated_survival_metric(fn, direction) new_static_survival_metric(fn, direction)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct a new metric function — new-metric","text":"fn function. metric function attach metric-specific class direction attribute . direction string. One : \"maximize\" \"minimize\" \"zero\"","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Create groupwise metrics — new_groupwise_metric","title":"Create groupwise metrics — new_groupwise_metric","text":"Groupwise metrics quantify disparity value metric across number groups. Groupwise metrics value zero indicate underlying metric equal across groups. yardstick defines several common fairness metrics using function, demographic_parity(), equal_opportunity(), equalized_odds().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create groupwise metrics — new_groupwise_metric","text":"","code":"new_groupwise_metric(fn, name, aggregate, direction = \"minimize\")"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create groupwise metrics — new_groupwise_metric","text":"fn yardstick metric function metric set. name name metric place .metric column output. aggregate function summarize generated metric set results. function takes metric set results first argument returns single numeric giving .estimate value output. See Value Examples sections example uses. direction string. One : \"maximize\" \"minimize\" \"zero\"","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create groupwise metrics — new_groupwise_metric","text":"function function factory; output function. , functions function outputs also function factories. explicitly, looks like: outputted dem_parity function takes one argument, , indicating data-masked variable giving sensitive feature. called argument, dem_parity return yardstick metric function like : Note dem_parity take arguments , thus knows nothing data applied column name \"gender\" . output dem_parity_by_gender metric function takes arguments function supplied fn, case detection_prevalence. thus interface like yardstick function except look \"gender\" column data supplied. addition examples , see documentation return value fairness metrics like demographic_parity(), equal_opportunity(), equalized_odds() learn output function can used.","code":"# a function with similar implementation to `demographic_parity()`: diff_range <- function(x) {diff(range(x$.estimate))} dem_parity <- new_groupwise_metric( fn = detection_prevalence, name = \"dem_parity\", aggregate = diff_range ) dem_parity_by_gender <- dem_parity(gender)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Create groupwise metrics — new_groupwise_metric","text":"Note yardstick metrics group-aware , passed grouped data, return metric values calculated group. passed grouped data, groupwise metrics also return metric values group, metric values calculated first additionally grouping variable passed summarizing per-group metric estimates across groups using function passed aggregate argument. Learn grouping behavior yardstick using vignette(\"grouping\", \"yardstick\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"relevant-group-level","dir":"Reference","previous_headings":"","what":"Relevant Group Level","title":"Create groupwise metrics — new_groupwise_metric","text":"Additional arguments can passed function outputted function function outputs. : finer control groups treated, use aggregate argument.","code":"res_fairness <- new_groupwise_metric(...) res_by <- res_fairness(by) res_by(..., additional_arguments_to_aggregate = TRUE)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create groupwise metrics — new_groupwise_metric","text":"","code":"data(hpc_cv) # `demographic_parity`, among other fairness metrics, # is generated with `new_groupwise_metric()`: diff_range <- function(x) {diff(range(x$.estimate))} demographic_parity_ <- new_groupwise_metric( fn = detection_prevalence, name = \"demographic_parity\", aggregate = diff_range ) m_set <- metric_set(demographic_parity_(Resample)) m_set(hpc_cv, truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 demographic_parity Resample macro 2.78e-17 # the `post` argument can be used to accommodate a wide # variety of parameterizations. to encode demographic # parity as a ratio inside of a difference, for example: ratio_range <- function(x, ...) { range <- range(x$.estimate) range[1] / range[2] } demographic_parity_ratio <- new_groupwise_metric( fn = detection_prevalence, name = \"demographic_parity_ratio\", aggregate = ratio_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":null,"dir":"Reference","previous_headings":"","what":"Negative predictive value — npv","title":"Negative predictive value — npv","text":"functions calculate npv() (negative predictive value) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), sens(), ppv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Negative predictive value — npv","text":"","code":"npv(data, ...) # S3 method for data.frame npv( data, truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) npv_vec( truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Negative predictive value — npv","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. prevalence numeric value rate \"positive\" class data. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Negative predictive value — npv","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. npv_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Negative predictive value — npv","text":"positive predictive value (ppv()) defined percent predicted positives actually positive negative predictive value (npv()) defined percent negative positives actually negative.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Negative predictive value — npv","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Negative predictive value — npv","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Negative predictive value — npv","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Negative predictive value — npv","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 2: predictive values,'' British Medical Journal, vol 309, 102.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Negative predictive value — npv","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Negative predictive value — npv","text":"","code":"# Two class data(\"two_class_example\") npv(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 npv binary 0.861 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% npv(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 npv macro 0.906 # Groups are respected hpc_cv %>% group_by(Resample) %>% npv(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 npv macro 0.906 #> 2 Fold02 npv macro 0.901 #> 3 Fold03 npv macro 0.917 #> 4 Fold04 npv macro 0.897 #> 5 Fold05 npv macro 0.897 #> 6 Fold06 npv macro 0.892 #> 7 Fold07 npv macro 0.882 #> 8 Fold08 npv macro 0.902 #> 9 Fold09 npv macro 0.879 #> 10 Fold10 npv macro 0.890 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% npv(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 npv macro_weighted 0.896 #> 2 Fold02 npv macro_weighted 0.890 #> 3 Fold03 npv macro_weighted 0.905 #> 4 Fold04 npv macro_weighted 0.878 #> 5 Fold05 npv macro_weighted 0.878 #> 6 Fold06 npv macro_weighted 0.871 #> 7 Fold07 npv macro_weighted 0.853 #> 8 Fold08 npv macro_weighted 0.885 #> 9 Fold09 npv macro_weighted 0.845 #> 10 Fold10 npv macro_weighted 0.864 # Vector version npv_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8609865 # Making Class2 the \"relevant\" level npv_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8194946"},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":null,"dir":"Reference","previous_headings":"","what":"Liver Pathology Data — pathology","title":"Liver Pathology Data — pathology","text":"Liver Pathology Data","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Liver Pathology Data — pathology","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Liver Pathology Data — pathology","text":"pathology data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Liver Pathology Data — pathology","text":"data results x-ray examination determine whether liver abnormal (scan column) versus extensive pathology results approximate truth (pathology).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Liver Pathology Data — pathology","text":"","code":"data(pathology) str(pathology) #> 'data.frame':\t344 obs. of 2 variables: #> $ pathology: Factor w/ 2 levels \"abnorm\",\"norm\": 1 1 1 1 1 1 1 1 1 1 ... #> $ scan : Factor w/ 2 levels \"abnorm\",\"norm\": 1 1 1 1 1 1 1 1 1 1 ..."},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean log loss for Poisson data — poisson_log_loss","title":"Mean log loss for Poisson data — poisson_log_loss","text":"Calculate loss function Poisson distribution.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean log loss for Poisson data — poisson_log_loss","text":"","code":"poisson_log_loss(data, ...) # S3 method for data.frame poisson_log_loss(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) poisson_log_loss_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean log loss for Poisson data — poisson_log_loss","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true counts (integer). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, integer vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean log loss for Poisson data — poisson_log_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. poisson_log_loss_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean log loss for Poisson data — poisson_log_loss","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean log loss for Poisson data — poisson_log_loss","text":"","code":"count_truth <- c(2L, 7L, 1L, 1L, 0L, 3L) count_pred <- c(2.14, 5.35, 1.65, 1.56, 1.3, 2.71) count_results <- dplyr::tibble(count = count_truth, pred = count_pred) # Supply truth and predictions as bare column names poisson_log_loss(count_results, count, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 poisson_log_loss standard 1.42"},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":null,"dir":"Reference","previous_headings":"","what":"Positive predictive value — ppv","title":"Positive predictive value — ppv","text":"functions calculate ppv() (positive predictive value) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), sens(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Positive predictive value — ppv","text":"","code":"ppv(data, ...) # S3 method for data.frame ppv( data, truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) ppv_vec( truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Positive predictive value — ppv","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. prevalence numeric value rate \"positive\" class data. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Positive predictive value — ppv","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. ppv_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Positive predictive value — ppv","text":"positive predictive value (ppv()) defined percent predicted positives actually positive negative predictive value (npv()) defined percent negative positives actually negative.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Positive predictive value — ppv","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Positive predictive value — ppv","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Positive predictive value — ppv","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Positive predictive value — ppv","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 2: predictive values,'' British Medical Journal, vol 309, 102.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Positive predictive value — ppv","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Positive predictive value — ppv","text":"","code":"# Two class data(\"two_class_example\") ppv(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv binary 0.819 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% ppv(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv macro 0.637 # Groups are respected hpc_cv %>% group_by(Resample) %>% ppv(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 ppv macro 0.637 #> 2 Fold02 ppv macro 0.603 #> 3 Fold03 ppv macro 0.706 #> 4 Fold04 ppv macro 0.658 #> 5 Fold05 ppv macro 0.651 #> 6 Fold06 ppv macro 0.626 #> 7 Fold07 ppv macro 0.562 #> 8 Fold08 ppv macro 0.652 #> 9 Fold09 ppv macro 0.605 #> 10 Fold10 ppv macro 0.625 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% ppv(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 ppv macro_weighted 0.697 #> 2 Fold02 ppv macro_weighted 0.690 #> 3 Fold03 ppv macro_weighted 0.752 #> 4 Fold04 ppv macro_weighted 0.690 #> 5 Fold05 ppv macro_weighted 0.705 #> 6 Fold06 ppv macro_weighted 0.682 #> 7 Fold07 ppv macro_weighted 0.649 #> 8 Fold08 ppv macro_weighted 0.702 #> 9 Fold09 ppv macro_weighted 0.661 #> 10 Fold10 ppv macro_weighted 0.683 # Vector version ppv_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8194946 # Making Class2 the \"relevant\" level ppv_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8609865 # But what if we think that Class 1 only occurs 40% of the time? ppv(two_class_example, truth, predicted, prevalence = 0.40) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv binary 0.740"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the precision recall curve — pr_auc","title":"Area under the precision recall curve — pr_auc","text":"pr_auc() metric computes area precision recall curve. See pr_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the precision recall curve — pr_auc","text":"","code":"pr_auc(data, ...) # S3 method for data.frame pr_auc( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) pr_auc_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the precision recall curve — pr_auc","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the precision recall curve — pr_auc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. pr_auc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the precision recall curve — pr_auc","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the precision recall curve — pr_auc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the precision recall curve — pr_auc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the precision recall curve — pr_auc","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. pr_auc(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc binary 0.946 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% pr_auc(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc macro 0.611 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% pr_auc(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc macro 0.611 # Groups are respected hpc_cv %>% group_by(Resample) %>% pr_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 pr_auc macro 0.611 #> 2 Fold02 pr_auc macro 0.620 #> 3 Fold03 pr_auc macro 0.689 #> 4 Fold04 pr_auc macro 0.680 #> 5 Fold05 pr_auc macro 0.620 #> 6 Fold06 pr_auc macro 0.650 #> 7 Fold07 pr_auc macro 0.607 #> 8 Fold08 pr_auc macro 0.650 #> 9 Fold09 pr_auc macro 0.628 #> 10 Fold10 pr_auc macro 0.603 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% pr_auc(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 pr_auc macro_weighted 0.746 #> 2 Fold02 pr_auc macro_weighted 0.743 #> 3 Fold03 pr_auc macro_weighted 0.789 #> 4 Fold04 pr_auc macro_weighted 0.754 #> 5 Fold05 pr_auc macro_weighted 0.737 #> 6 Fold06 pr_auc macro_weighted 0.743 #> 7 Fold07 pr_auc macro_weighted 0.748 #> 8 Fold08 pr_auc macro_weighted 0.756 #> 9 Fold09 pr_auc macro_weighted 0.711 #> 10 Fold10 pr_auc macro_weighted 0.737 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") pr_auc_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.6109931"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Precision recall curve — pr_curve","title":"Precision recall curve — pr_curve","text":"pr_curve() constructs full precision recall curve returns tibble. See pr_auc() area precision recall curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Precision recall curve — pr_curve","text":"","code":"pr_curve(data, ...) # S3 method for data.frame pr_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Precision recall curve — pr_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Precision recall curve — pr_curve","text":"tibble class pr_df pr_grouped_df columns .threshold, recall, precision.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Precision recall curve — pr_curve","text":"pr_curve() computes precision every unique value probability column (addition infinity). ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Precision recall curve — pr_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Precision recall curve — pr_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Precision recall curve — pr_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Precision recall curve — pr_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. pr_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 3 #> .threshold recall precision #> #> 1 Inf 0 1 #> 2 1.00 0.00388 1 #> 3 1.00 0.00775 1 #> 4 1.00 0.0116 1 #> 5 1.00 0.0155 1 #> 6 1.00 0.0194 1 #> 7 1.00 0.0233 1 #> 8 1.00 0.0271 1 #> 9 1.00 0.0310 1 #> 10 1.00 0.0349 1 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) pr_curve(two_class_example, truth, Class1) %>% ggplot(aes(x = recall, y = precision)) + geom_path() + coord_equal() + theme_bw() # Or use autoplot autoplot(pr_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% pr_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% pr_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":null,"dir":"Reference","previous_headings":"","what":"Precision — precision","title":"Precision — precision","text":"functions calculate precision() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions recall() f_meas().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Precision — precision","text":"","code":"precision(data, ...) # S3 method for data.frame precision( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) precision_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Precision — precision","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Precision — precision","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. precision_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Precision — precision","text":"precision percentage predicted truly relevant results total number predicted relevant results characterizes \"purity retrieval performance\" (Buckland Gey, 1994). denominator calculation 0, precision undefined. happens # true_positive = 0 # false_positive = 0 true, mean predicted events. computing binary precision, NA value returned warning. computing multiclass precision, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Precision — precision","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Precision — precision","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Precision — precision","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Precision — precision","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Precision — precision","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Precision — precision","text":"","code":"# Two class data(\"two_class_example\") precision(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision binary 0.819 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% precision(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision macro 0.637 # Groups are respected hpc_cv %>% group_by(Resample) %>% precision(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 precision macro 0.637 #> 2 Fold02 precision macro 0.603 #> 3 Fold03 precision macro 0.706 #> 4 Fold04 precision macro 0.658 #> 5 Fold05 precision macro 0.651 #> 6 Fold06 precision macro 0.626 #> 7 Fold07 precision macro 0.562 #> 8 Fold08 precision macro 0.652 #> 9 Fold09 precision macro 0.605 #> 10 Fold10 precision macro 0.625 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% precision(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 precision macro_weighted 0.697 #> 2 Fold02 precision macro_weighted 0.690 #> 3 Fold03 precision macro_weighted 0.752 #> 4 Fold04 precision macro_weighted 0.690 #> 5 Fold05 precision macro_weighted 0.705 #> 6 Fold06 precision macro_weighted 0.682 #> 7 Fold07 precision macro_weighted 0.649 #> 8 Fold08 precision macro_weighted 0.702 #> 9 Fold09 precision macro_weighted 0.661 #> 10 Fold10 precision macro_weighted 0.683 # Vector version precision_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8194946 # Making Class2 the \"relevant\" level precision_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8609865"},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":null,"dir":"Reference","previous_headings":"","what":"Recall — recall","title":"Recall — recall","text":"functions calculate recall() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions precision() f_meas().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Recall — recall","text":"","code":"recall(data, ...) # S3 method for data.frame recall( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) recall_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Recall — recall","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Recall — recall","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. recall_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Recall — recall","text":"recall (aka sensitivity) defined proportion relevant results number samples actually relevant. relevant results, recall defined value NA returned. denominator calculation 0, recall undefined. happens # true_positive = 0 # false_negative = 0 true, mean true events. computing binary recall, NA value returned warning. computing multiclass recall, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Recall — recall","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Recall — recall","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Recall — recall","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Recall — recall","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Recall — recall","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Recall — recall","text":"","code":"# Two class data(\"two_class_example\") recall(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 recall binary 0.880 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% recall(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 recall macro 0.548 # Groups are respected hpc_cv %>% group_by(Resample) %>% recall(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 recall macro 0.548 #> 2 Fold02 recall macro 0.541 #> 3 Fold03 recall macro 0.634 #> 4 Fold04 recall macro 0.570 #> 5 Fold05 recall macro 0.550 #> 6 Fold06 recall macro 0.540 #> 7 Fold07 recall macro 0.531 #> 8 Fold08 recall macro 0.584 #> 9 Fold09 recall macro 0.568 #> 10 Fold10 recall macro 0.537 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% recall(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 recall macro_weighted 0.726 #> 2 Fold02 recall macro_weighted 0.712 #> 3 Fold03 recall macro_weighted 0.758 #> 4 Fold04 recall macro_weighted 0.712 #> 5 Fold05 recall macro_weighted 0.712 #> 6 Fold06 recall macro_weighted 0.697 #> 7 Fold07 recall macro_weighted 0.675 #> 8 Fold08 recall macro_weighted 0.721 #> 9 Fold09 recall macro_weighted 0.673 #> 10 Fold10 recall macro_weighted 0.699 # Vector version recall_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.879845 # Making Class2 the \"relevant\" level recall_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.7933884"},{"path":"https://yardstick.tidymodels.org/dev/reference/reexports.html","id":null,"dir":"Reference","previous_headings":"","what":"Objects exported from other packages — reexports","title":"Objects exported from other packages — reexports","text":"objects imported packages. Follow links see documentation. generics tidy","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":null,"dir":"Reference","previous_headings":"","what":"Root mean squared error — rmse","title":"Root mean squared error — rmse","text":"Calculate root mean squared error. rmse() metric units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Root mean squared error — rmse","text":"","code":"rmse(data, ...) # S3 method for data.frame rmse(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rmse_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Root mean squared error — rmse","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Root mean squared error — rmse","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rmse_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Root mean squared error — rmse","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Root mean squared error — rmse","text":"","code":"# Supply truth and predictions as bare column names rmse(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rmse(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rmse standard 0.715 #> 2 10 rmse standard 0.673 #> 3 2 rmse standard 0.716 #> 4 3 rmse standard 0.644 #> 5 4 rmse standard 0.737 #> 6 5 rmse standard 0.675 #> 7 6 rmse standard 0.807 #> 8 7 rmse standard 0.801 #> 9 8 rmse standard 0.635 #> 10 9 rmse standard 0.692 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.710"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the receiver operator curve — roc_auc","title":"Area under the receiver operator curve — roc_auc","text":"roc_auc() metric computes area ROC curve. See roc_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the receiver operator curve — roc_auc","text":"","code":"roc_auc(data, ...) # S3 method for data.frame roc_auc( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list() ) roc_auc_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the receiver operator curve — roc_auc","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"hand_till\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. others general methods calculating multiclass metrics. default automatically choose \"binary\" truth binary, \"hand_till\" truth >2 levels case_weights specified, \"macro\" truth >2 levels case_weights specified (case \"hand_till\" well-defined). na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the receiver operator curve — roc_auc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_auc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Area under the receiver operator curve — roc_auc","text":"Generally, ROC AUC value 0.5 1, 1 perfect prediction model. value 0 0.5, implies meaningful information model, applied incorrectly opposite model predicts result AUC >0.5. Note combine estimator = \"hand_till\" case_weights.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the receiver operator curve — roc_auc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the receiver operator curve — roc_auc","text":"default multiclass method computing roc_auc() use method Hand, Till, (2001). Unlike macro-averaging, method insensitive class distributions like binary ROC AUC case. Additionally, multiclass techniques return NA levels truth occur zero times actual data, Hand-Till method simply ignore levels averaging calculation, warning. Macro macro-weighted averaging still provided, even though default. fact, macro-weighted averaging corresponds definition multiclass AUC given Provost Domingos (2001).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the receiver operator curve — roc_auc","text":"Hand, Till (2001). \"Simple Generalisation Area ROC Curve Multiple Class Classification Problems\". Machine Learning. Vol 45, Iss 2, pp 171-186. Fawcett (2005). \"introduction ROC analysis\". Pattern Recognition Letters. 27 (2006), pp 861-874. Provost, F., Domingos, P., 2001. \"Well-trained PETs: Improving probability estimation trees\", CeDER Working Paper #-00-04, Stern School Business, New York University, NY, NY 10012.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the receiver operator curve — roc_auc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the receiver operator curve — roc_auc","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. roc_auc(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc binary 0.939 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_auc(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc hand_till 0.813 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_auc(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc hand_till 0.813 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc hand_till 0.813 #> 2 Fold02 roc_auc hand_till 0.817 #> 3 Fold03 roc_auc hand_till 0.869 #> 4 Fold04 roc_auc hand_till 0.849 #> 5 Fold05 roc_auc hand_till 0.811 #> 6 Fold06 roc_auc hand_till 0.836 #> 7 Fold07 roc_auc hand_till 0.825 #> 8 Fold08 roc_auc hand_till 0.846 #> 9 Fold09 roc_auc hand_till 0.828 #> 10 Fold10 roc_auc hand_till 0.812 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc macro_weighted 0.880 #> 2 Fold02 roc_auc macro_weighted 0.873 #> 3 Fold03 roc_auc macro_weighted 0.906 #> 4 Fold04 roc_auc macro_weighted 0.867 #> 5 Fold05 roc_auc macro_weighted 0.866 #> 6 Fold06 roc_auc macro_weighted 0.865 #> 7 Fold07 roc_auc macro_weighted 0.868 #> 8 Fold08 roc_auc macro_weighted 0.865 #> 9 Fold09 roc_auc macro_weighted 0.841 #> 10 Fold10 roc_auc macro_weighted 0.869 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_auc_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8131924"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Compute area ROC survival curve using predicted survival probabilities corresponds different time points.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"","code":"roc_auc_survival(data, ...) # S3 method for data.frame roc_auc_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL) roc_auc_survival_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. roc_auc_survival_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes area ROC curve. account censoring, inverse probability censoring weights (IPCW) used calculations. See equation 7 section 4.3 Blanche al (2013) details. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument. Larger values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Blanche, P., Dartigues, J.-F. Jacqmin-Gadda, H. (2013), Review comparison ROC curve estimators time-dependent outcome marker-dependent censoring. Biom. J., 55: 687-704. Graf, E., Schmoor, C., Sauerbrei, W. Schumacher, M. (1999), Assessment comparison prognostic classification schemes survival data. Statist. Med., 18: 2529-2545.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"","code":"library(dplyr) lung_surv %>% roc_auc_survival( truth = surv_obj, .pred ) #> # A tibble: 5 × 4 #> .metric .estimator .eval_time .estimate #> #> 1 roc_auc_survival standard 100 0.659 #> 2 roc_auc_survival standard 200 0.679 #> 3 roc_auc_survival standard 300 0.688 #> 4 roc_auc_survival standard 400 0.648 #> 5 roc_auc_survival standard 500 0.662"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"roc_aunp() multiclass metric computes area ROC curve class rest, using priori class distribution. equivalent roc_auc(estimator = \"macro_weighted\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"","code":"roc_aunp(data, ...) # S3 method for data.frame roc_aunp(data, truth, ..., na_rm = TRUE, case_weights = NULL, options = list()) roc_aunp_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. many columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_aunp_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"multiclass method computing area ROC curve uses priori class distribution equivalent roc_auc(estimator = \"macro_weighted\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"Ferri, C., Hernández-Orallo, J., & Modroiu, R. (2009). \"experimental comparison performance measures classification\". Pattern Recognition Letters. 30 (1), pp 27-38.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"Julia Silge","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"","code":"# Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_aunp(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunp macro 0.880 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_aunp(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunp macro 0.880 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_aunp(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_aunp macro 0.880 #> 2 Fold02 roc_aunp macro 0.873 #> 3 Fold03 roc_aunp macro 0.906 #> 4 Fold04 roc_aunp macro 0.867 #> 5 Fold05 roc_aunp macro 0.866 #> 6 Fold06 roc_aunp macro 0.865 #> 7 Fold07 roc_aunp macro 0.868 #> 8 Fold08 roc_aunp macro 0.865 #> 9 Fold09 roc_aunp macro 0.841 #> 10 Fold10 roc_aunp macro 0.869 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_aunp_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8795121"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"roc_aunu() multiclass metric computes area ROC curve class rest, using uniform class distribution. equivalent roc_auc(estimator = \"macro\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"","code":"roc_aunu(data, ...) # S3 method for data.frame roc_aunu(data, truth, ..., na_rm = TRUE, case_weights = NULL, options = list()) roc_aunu_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. many columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_aunu_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"multiclass method computing area ROC curve uses uniform class distribution equivalent roc_auc(estimator = \"macro\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"Ferri, C., Hernández-Orallo, J., & Modroiu, R. (2009). \"experimental comparison performance measures classification\". Pattern Recognition Letters. 30 (1), pp 27-38.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"Julia Silge","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"","code":"# Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_aunu(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunu macro 0.871 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_aunu(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunu macro 0.871 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_aunu(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_aunu macro 0.871 #> 2 Fold02 roc_aunu macro 0.863 #> 3 Fold03 roc_aunu macro 0.898 #> 4 Fold04 roc_aunu macro 0.874 #> 5 Fold05 roc_aunu macro 0.865 #> 6 Fold06 roc_aunu macro 0.877 #> 7 Fold07 roc_aunu macro 0.865 #> 8 Fold08 roc_aunu macro 0.873 #> 9 Fold09 roc_aunu macro 0.855 #> 10 Fold10 roc_aunu macro 0.865 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_aunu_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8714461"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Receiver operator curve — roc_curve","title":"Receiver operator curve — roc_curve","text":"roc_curve() constructs full ROC curve returns tibble. See roc_auc() area ROC curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Receiver operator curve — roc_curve","text":"","code":"roc_curve(data, ...) # S3 method for data.frame roc_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Receiver operator curve — roc_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Receiver operator curve — roc_curve","text":"tibble class roc_df roc_grouped_df columns .threshold, specificity, sensitivity.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Receiver operator curve — roc_curve","text":"roc_curve() computes sensitivity every unique value probability column (addition infinity minus infinity). ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Receiver operator curve — roc_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Receiver operator curve — roc_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Receiver operator curve — roc_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Receiver operator curve — roc_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. roc_curve(two_class_example, truth, Class1) #> # A tibble: 502 × 3 #> .threshold specificity sensitivity #> #> 1 -Inf 0 1 #> 2 1.79e-7 0 1 #> 3 4.50e-6 0.00413 1 #> 4 5.81e-6 0.00826 1 #> 5 5.92e-6 0.0124 1 #> 6 1.22e-5 0.0165 1 #> 7 1.40e-5 0.0207 1 #> 8 1.43e-5 0.0248 1 #> 9 2.38e-5 0.0289 1 #> 10 3.30e-5 0.0331 1 #> # ℹ 492 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) roc_curve(two_class_example, truth, Class1) %>% ggplot(aes(x = 1 - specificity, y = sensitivity)) + geom_path() + geom_abline(lty = 3) + coord_equal() + theme_bw() # Or use autoplot autoplot(roc_curve(two_class_example, truth, Class1)) if (FALSE) { # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% roc_curve(obs, VF:L) %>% autoplot() }"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Compute ROC survival curve using predicted survival probabilities corresponds different time points.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"","code":"roc_curve_survival(data, ...) # S3 method for data.frame roc_curve_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL)"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"tibble class roc_survival_df, grouped_roc_survival_df columns .threshold, sensitivity, specificity, .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes ROC curve. account censoring, inverse probability censoring weights (IPCW) used calculations. See equation 7 section 4.3 Blanche al (2013) details. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Blanche, P., Dartigues, J.-F. Jacqmin-Gadda, H. (2013), Review comparison ROC curve estimators time-dependent outcome marker-dependent censoring. Biom. J., 55: 687-704. Graf, E., Schmoor, C., Sauerbrei, W. Schumacher, M. (1999), Assessment comparison prognostic classification schemes survival data. Statist. Med., 18: 2529-2545.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"","code":"result <- roc_curve_survival( lung_surv, truth = surv_obj, .pred ) result #> # A tibble: 650 × 4 #> .threshold sensitivity specificity .eval_time #> #> 1 -Inf 0 1 100 #> 2 0.615 0 0.995 100 #> 3 0.723 0.0334 0.985 100 #> 4 0.727 0.133 0.985 100 #> 5 0.730 0.167 0.985 100 #> 6 0.732 0.233 0.980 100 #> 7 0.739 0.233 0.969 100 #> 8 0.741 0.267 0.964 100 #> 9 0.746 0.267 0.959 100 #> 10 0.748 0.267 0.954 100 #> # ℹ 640 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) result %>% mutate(.eval_time = format(.eval_time)) %>% ggplot(aes( x = 1 - specificity, y = sensitivity, group = .eval_time, col = .eval_time )) + geom_step(direction = \"hv\") + geom_abline(lty = 3) + coord_equal() + theme_bw() # Or use autoplot autoplot(result)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":null,"dir":"Reference","previous_headings":"","what":"Ratio of performance to deviation — rpd","title":"Ratio of performance to deviation — rpd","text":"functions appropriate cases model outcome numeric. ratio performance deviation (rpd()) ratio performance inter-quartile (rpiq()) measures consistency/correlation observed predicted values (accuracy).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ratio of performance to deviation — rpd","text":"","code":"rpd(data, ...) # S3 method for data.frame rpd(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rpd_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ratio of performance to deviation — rpd","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ratio of performance to deviation — rpd","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rpd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Ratio of performance to deviation — rpd","text":"field spectroscopy particular, ratio performance deviation (RPD) used standard way report quality model. ratio standard deviation variable standard error prediction variable given model. However, systematic use criticized several authors, since using standard deviation represent spread variable can misleading skewed dataset. ratio performance inter-quartile introduced Bellon-Maurel et al. (2010) address issues, generalise RPD non-normally distributed variables.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ratio of performance to deviation — rpd","text":"Williams, P.C. (1987) Variables affecting near-infrared reflectance spectroscopic analysis. : Near Infrared Technology Agriculture Food Industries. 1st Ed. P.Williams K.Norris, Eds. . Cereal Assoc. Cereal Chem., St. Paul, MN. Bellon-Maurel, V., Fernandez-Ahumada, E., Palagos, B., Roger, J.M. McBratney, ., (2010). Critical review chemometric indicators commonly used assessing quality prediction soil attributes NIR spectroscopy. TrAC Trends Analytical Chemistry, 29(9), pp.1073-1081.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Ratio of performance to deviation — rpd","text":"Pierre Roudier","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ratio of performance to deviation — rpd","text":"","code":"# Supply truth and predictions as bare column names rpd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rpd standard 2.88 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rpd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rpd standard 2.78 #> 2 10 rpd standard 2.87 #> 3 2 rpd standard 3.04 #> 4 3 rpd standard 3.41 #> 5 4 rpd standard 3.02 #> 6 5 rpd standard 2.66 #> 7 6 rpd standard 2.81 #> 8 7 rpd standard 2.61 #> 9 8 rpd standard 3.45 #> 10 9 rpd standard 2.93 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 2.96"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":null,"dir":"Reference","previous_headings":"","what":"Ratio of performance to inter-quartile — rpiq","title":"Ratio of performance to inter-quartile — rpiq","text":"functions appropriate cases model outcome numeric. ratio performance deviation (rpd()) ratio performance inter-quartile (rpiq()) measures consistency/correlation observed predicted values (accuracy).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ratio of performance to inter-quartile — rpiq","text":"","code":"rpiq(data, ...) # S3 method for data.frame rpiq(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rpiq_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ratio of performance to inter-quartile — rpiq","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ratio of performance to inter-quartile — rpiq","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rpd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Ratio of performance to inter-quartile — rpiq","text":"field spectroscopy particular, ratio performance deviation (RPD) used standard way report quality model. ratio standard deviation variable standard error prediction variable given model. However, systematic use criticized several authors, since using standard deviation represent spread variable can misleading skewed dataset. ratio performance inter-quartile introduced Bellon-Maurel et al. (2010) address issues, generalise RPD non-normally distributed variables.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ratio of performance to inter-quartile — rpiq","text":"Williams, P.C. (1987) Variables affecting near-infrared reflectance spectroscopic analysis. : Near Infrared Technology Agriculture Food Industries. 1st Ed. P.Williams K.Norris, Eds. . Cereal Assoc. Cereal Chem., St. Paul, MN. Bellon-Maurel, V., Fernandez-Ahumada, E., Palagos, B., Roger, J.M. McBratney, ., (2010). Critical review chemometric indicators commonly used assessing quality prediction soil attributes NIR spectroscopy. TrAC Trends Analytical Chemistry, 29(9), pp.1073-1081.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Ratio of performance to inter-quartile — rpiq","text":"Pierre Roudier","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ratio of performance to inter-quartile — rpiq","text":"","code":"# Supply truth and predictions as bare column names rpd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rpd standard 2.88 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rpd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rpd standard 2.78 #> 2 10 rpd standard 2.87 #> 3 2 rpd standard 3.04 #> 4 3 rpd standard 3.41 #> 5 4 rpd standard 3.02 #> 6 5 rpd standard 2.66 #> 7 6 rpd standard 2.81 #> 8 7 rpd standard 2.61 #> 9 8 rpd standard 3.45 #> 10 9 rpd standard 2.93 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 2.96"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":null,"dir":"Reference","previous_headings":"","what":"R squared — rsq","title":"R squared — rsq","text":"Calculate coefficient determination using correlation. traditional measure R squared, see rsq_trad().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"R squared — rsq","text":"","code":"rsq(data, ...) # S3 method for data.frame rsq(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rsq_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"R squared — rsq","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"R squared — rsq","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rsq_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"R squared — rsq","text":"two estimates coefficient determination, rsq() rsq_trad(), differ formula. former guarantees value (0, 1) latter can generate inaccurate values model non-informative (see examples). measures consistency/correlation accuracy. rsq() simply squared correlation truth estimate. rsq() internally computes correlation, either truth estimate constant can result divide zero error. cases, warning thrown NA returned. can occur model predicts single value samples. example, regularized model eliminates predictors except intercept . Another example CART model contains splits.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"R squared — rsq","text":"Kvalseth. Cautionary note \\(R^2\\). American Statistician (1985) vol. 39 (4) pp. 279-285.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"R squared — rsq","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"R squared — rsq","text":"","code":"# Supply truth and predictions as bare column names rsq(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.879 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rsq(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rsq standard 0.874 #> 2 10 rsq standard 0.879 #> 3 2 rsq standard 0.891 #> 4 3 rsq standard 0.916 #> 5 4 rsq standard 0.892 #> 6 5 rsq standard 0.858 #> 7 6 rsq standard 0.873 #> 8 7 rsq standard 0.852 #> 9 8 rsq standard 0.915 #> 10 9 rsq standard 0.884 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.883 # With uninformitive data, the traditional version of R^2 can return # negative values. set.seed(2291) solubility_test$randomized <- sample(solubility_test$prediction) rsq(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.00199 rsq_trad(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard -1.01 # A constant `truth` or `estimate` vector results in a warning from # a divide by zero error in the correlation calculation. # `NA` will be returned in these cases. truth <- c(1, 2) estimate <- c(1, 1) rsq_vec(truth, estimate) #> Warning: A correlation computation is required, but `estimate` is constant and has #> 0 standard deviation, resulting in a divide by 0 error. `NA` will be #> returned. #> [1] NA"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":null,"dir":"Reference","previous_headings":"","what":"R squared - traditional — rsq_trad","title":"R squared - traditional — rsq_trad","text":"Calculate coefficient determination using traditional definition R squared using sum squares. measure R squared strictly (0, 1), see rsq().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"R squared - traditional — rsq_trad","text":"","code":"rsq_trad(data, ...) # S3 method for data.frame rsq_trad(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rsq_trad_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"R squared - traditional — rsq_trad","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"R squared - traditional — rsq_trad","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rsq_trad_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"R squared - traditional — rsq_trad","text":"two estimates coefficient determination, rsq() rsq_trad(), differ formula. former guarantees value (0, 1) latter can generate inaccurate values model non-informative (see examples). measures consistency/correlation accuracy.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"R squared - traditional — rsq_trad","text":"Kvalseth. Cautionary note \\(R^2\\). American Statistician (1985) vol. 39 (4) pp. 279-285.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"R squared - traditional — rsq_trad","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"R squared - traditional — rsq_trad","text":"","code":"# Supply truth and predictions as bare column names rsq_trad(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard 0.879 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rsq_trad(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rsq_trad standard 0.870 #> 2 10 rsq_trad standard 0.878 #> 3 2 rsq_trad standard 0.891 #> 4 3 rsq_trad standard 0.913 #> 5 4 rsq_trad standard 0.889 #> 6 5 rsq_trad standard 0.857 #> 7 6 rsq_trad standard 0.872 #> 8 7 rsq_trad standard 0.852 #> 9 8 rsq_trad standard 0.915 #> 10 9 rsq_trad standard 0.883 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.882 # With uninformitive data, the traditional version of R^2 can return # negative values. set.seed(2291) solubility_test$randomized <- sample(solubility_test$prediction) rsq(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.00199 rsq_trad(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard -1.01"},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":null,"dir":"Reference","previous_headings":"","what":"Sensitivity — sens","title":"Sensitivity — sens","text":"functions calculate sens() (sensitivity) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), ppv(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Sensitivity — sens","text":"","code":"sens(data, ...) # S3 method for data.frame sens( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sens_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sensitivity(data, ...) # S3 method for data.frame sensitivity( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sensitivity_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Sensitivity — sens","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Sensitivity — sens","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. sens_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Sensitivity — sens","text":"sensitivity (sens()) defined proportion positive results number samples actually positive. denominator calculation 0, sensitivity undefined. happens # true_positive = 0 # false_negative = 0 true, mean true events. computing binary sensitivity, NA value returned warning. computing multiclass sensitivity, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Sensitivity — sens","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Sensitivity — sens","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Sensitivity — sens","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Sensitivity — sens","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Sensitivity — sens","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Sensitivity — sens","text":"","code":"# Two class data(\"two_class_example\") sens(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 sens binary 0.880 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% sens(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 sens macro 0.548 # Groups are respected hpc_cv %>% group_by(Resample) %>% sens(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 sens macro 0.548 #> 2 Fold02 sens macro 0.541 #> 3 Fold03 sens macro 0.634 #> 4 Fold04 sens macro 0.570 #> 5 Fold05 sens macro 0.550 #> 6 Fold06 sens macro 0.540 #> 7 Fold07 sens macro 0.531 #> 8 Fold08 sens macro 0.584 #> 9 Fold09 sens macro 0.568 #> 10 Fold10 sens macro 0.537 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% sens(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 sens macro_weighted 0.726 #> 2 Fold02 sens macro_weighted 0.712 #> 3 Fold03 sens macro_weighted 0.758 #> 4 Fold04 sens macro_weighted 0.712 #> 5 Fold05 sens macro_weighted 0.712 #> 6 Fold06 sens macro_weighted 0.697 #> 7 Fold07 sens macro_weighted 0.675 #> 8 Fold08 sens macro_weighted 0.721 #> 9 Fold09 sens macro_weighted 0.673 #> 10 Fold10 sens macro_weighted 0.699 # Vector version sens_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.879845 # Making Class2 the \"relevant\" level sens_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.7933884"},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":null,"dir":"Reference","previous_headings":"","what":"Symmetric mean absolute percentage error — smape","title":"Symmetric mean absolute percentage error — smape","text":"Calculate symmetric mean absolute percentage error. metric relative units.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Symmetric mean absolute percentage error — smape","text":"","code":"smape(data, ...) # S3 method for data.frame smape(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) smape_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Symmetric mean absolute percentage error — smape","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Symmetric mean absolute percentage error — smape","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. smape_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Symmetric mean absolute percentage error — smape","text":"implementation smape() \"usual definition\" denominator divided two.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Symmetric mean absolute percentage error — smape","text":"Max Kuhn, Riaz Hedayati","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Symmetric mean absolute percentage error — smape","text":"","code":"# Supply truth and predictions as bare column names smape(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 smape standard 36.7 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% smape(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 smape standard 33.5 #> 2 10 smape standard 28.9 #> 3 2 smape standard 41.6 #> 4 3 smape standard 28.4 #> 5 4 smape standard 37.0 #> 6 5 smape standard 31.8 #> 7 6 smape standard 48.3 #> 8 7 smape standard 27.8 #> 9 8 smape standard 35.6 #> 10 9 smape standard 30.2 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 34.3"},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Solubility Predictions from MARS Model — solubility_test","title":"Solubility Predictions from MARS Model — solubility_test","text":"Solubility Predictions MARS Model","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Solubility Predictions from MARS Model — solubility_test","text":"Kuhn, M., Johnson, K. (2013) Applied Predictive Modeling, Springer","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Solubility Predictions from MARS Model — solubility_test","text":"solubility_test data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Solubility Predictions from MARS Model — solubility_test","text":"solubility data Kuhn Johnson (2013), data test set results MARS model. observed solubility (column solubility) model results (prediction) contained data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Solubility Predictions from MARS Model — solubility_test","text":"","code":"data(solubility_test) str(solubility_test) #> 'data.frame':\t316 obs. of 2 variables: #> $ solubility: num 0.93 0.85 0.81 0.74 0.61 0.58 0.57 0.56 0.52 0.45 ... #> $ prediction: num 0.368 -0.15 -0.505 0.54 -0.479 ..."},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":null,"dir":"Reference","previous_headings":"","what":"Specificity — spec","title":"Specificity — spec","text":"functions calculate spec() (specificity) measurement system compared reference result (\"truth\" gold standard). Highly related functions sens(), ppv(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Specificity — spec","text":"","code":"spec(data, ...) # S3 method for data.frame spec( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) spec_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) specificity(data, ...) # S3 method for data.frame specificity( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) specificity_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Specificity — spec","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Specificity — spec","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. spec_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Specificity — spec","text":"specificity measures proportion negatives correctly identified negatives. denominator calculation 0, specificity undefined. happens # true_negative = 0 # false_positive = 0 true, mean true negatives. computing binary specificity, NA value returned warning. computing multiclass specificity, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Specificity — spec","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Specificity — spec","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Specificity — spec","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Specificity — spec","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Specificity — spec","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Specificity — spec","text":"","code":"# Two class data(\"two_class_example\") spec(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 spec binary 0.793 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% spec(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 spec macro 0.886 # Groups are respected hpc_cv %>% group_by(Resample) %>% spec(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 spec macro 0.886 #> 2 Fold02 spec macro 0.882 #> 3 Fold03 spec macro 0.899 #> 4 Fold04 spec macro 0.879 #> 5 Fold05 spec macro 0.881 #> 6 Fold06 spec macro 0.873 #> 7 Fold07 spec macro 0.866 #> 8 Fold08 spec macro 0.884 #> 9 Fold09 spec macro 0.867 #> 10 Fold10 spec macro 0.875 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% spec(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 spec macro_weighted 0.816 #> 2 Fold02 spec macro_weighted 0.815 #> 3 Fold03 spec macro_weighted 0.839 #> 4 Fold04 spec macro_weighted 0.803 #> 5 Fold05 spec macro_weighted 0.812 #> 6 Fold06 spec macro_weighted 0.795 #> 7 Fold07 spec macro_weighted 0.790 #> 8 Fold08 spec macro_weighted 0.814 #> 9 Fold09 spec macro_weighted 0.795 #> 10 Fold10 spec macro_weighted 0.801 # Vector version spec_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.7933884 # Making Class2 the \"relevant\" level spec_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.879845"},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":null,"dir":"Reference","previous_headings":"","what":"Summary Statistics for Confusion Matrices — summary.conf_mat","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"Various statistical summaries confusion matrices produced returned tibble. include shown help pages sens(), recall(), accuracy(), among others.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"","code":"# S3 method for conf_mat summary( object, prevalence = NULL, beta = 1, estimator = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"object object class conf_mat(). prevalence number (0, 1) prevalence (.e. prior) event. left default, data used derive value. beta numeric value used weight precision recall f_meas(). estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. ... currently used.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"tibble containing various classification metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"","code":"data(\"two_class_example\") cmat <- conf_mat(two_class_example, truth = \"truth\", estimate = \"predicted\") summary(cmat) #> # A tibble: 13 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 sens binary 0.880 #> 4 spec binary 0.793 #> 5 ppv binary 0.819 #> 6 npv binary 0.861 #> 7 mcc binary 0.677 #> 8 j_index binary 0.673 #> 9 bal_accuracy binary 0.837 #> 10 detection_prevalence binary 0.554 #> 11 precision binary 0.819 #> 12 recall binary 0.880 #> 13 f_meas binary 0.849 summary(cmat, prevalence = 0.70) #> # A tibble: 13 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 sens binary 0.880 #> 4 spec binary 0.793 #> 5 ppv binary 0.909 #> 6 npv binary 0.739 #> 7 mcc binary 0.677 #> 8 j_index binary 0.673 #> 9 bal_accuracy binary 0.837 #> 10 detection_prevalence binary 0.554 #> 11 precision binary 0.819 #> 12 recall binary 0.880 #> 13 f_meas binary 0.849 library(dplyr) library(tidyr) data(\"hpc_cv\") # Compute statistics per resample then summarize all_metrics <- hpc_cv %>% group_by(Resample) %>% conf_mat(obs, pred) %>% mutate(summary_tbl = lapply(conf_mat, summary)) %>% unnest(summary_tbl) all_metrics %>% group_by(.metric) %>% summarise( mean = mean(.estimate, na.rm = TRUE), sd = sd(.estimate, na.rm = TRUE) ) #> # A tibble: 13 × 3 #> .metric mean sd #> #> 1 accuracy 0.709 2.47e- 2 #> 2 bal_accuracy 0.720 1.92e- 2 #> 3 detection_prevalence 0.25 9.25e-18 #> 4 f_meas 0.569 3.46e- 2 #> 5 j_index 0.439 3.85e- 2 #> 6 kap 0.508 4.10e- 2 #> 7 mcc 0.515 4.16e- 2 #> 8 npv 0.896 1.11e- 2 #> 9 ppv 0.633 3.87e- 2 #> 10 precision 0.633 3.87e- 2 #> 11 recall 0.560 3.09e- 2 #> 12 sens 0.560 3.09e- 2 #> 13 spec 0.879 9.67e- 3"},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":null,"dir":"Reference","previous_headings":"","what":"Two Class Predictions — two_class_example","title":"Two Class Predictions — two_class_example","text":"Two Class Predictions","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Two Class Predictions — two_class_example","text":"two_class_example data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Two Class Predictions — two_class_example","text":"data test set form model built two classes (\"Class1\" \"Class2\"). columns true predicted classes column probabilities class.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Two Class Predictions — two_class_example","text":"","code":"data(two_class_example) str(two_class_example) #> 'data.frame':\t500 obs. of 4 variables: #> $ truth : Factor w/ 2 levels \"Class1\",\"Class2\": 2 1 2 1 2 1 1 1 2 2 ... #> $ Class1 : num 0.00359 0.67862 0.11089 0.73516 0.01624 ... #> $ Class2 : num 0.996 0.321 0.889 0.265 0.984 ... #> $ predicted: Factor w/ 2 levels \"Class1\",\"Class2\": 2 1 2 1 2 1 1 1 2 2 ... # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section in any classification function (such as `?pr_auc`) to see how # to change this. levels(hpc_cv$obs) #> [1] \"VF\" \"F\" \"M\" \"L\""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick-package.html","id":null,"dir":"Reference","previous_headings":"","what":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","title":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","text":"Tidy tools quantifying well model fits data set confusion matrices, class probability curve summaries, regression metrics (e.g., RMSE).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","text":"Maintainer: Emil Hvitfeldt emil.hvitfeldt@posit.co (ORCID) Authors: Max Kuhn max@posit.co Davis Vaughan davis@posit.co contributors: Posit Software, PBC [copyright holder, funder]","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for handling missing values in new metrics — yardstick_remove_missing","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"yardstick_remove_missing(), yardstick_any_missing() useful alongside metric-summarizers functions implementing new custom metrics. yardstick_remove_missing() removes observations contains missing values across, truth, estimate case_weights. yardstick_any_missing() returns FALSE missing values inputs.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"","code":"yardstick_remove_missing(truth, estimate, case_weights) yardstick_any_missing(truth, estimate, case_weights)"},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"truth, estimate Vectors length. case_weights vector length truth estimate, NULL case weights used.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-development-version","dir":"Changelog","previous_headings":"","what":"yardstick (development version)","title":"yardstick (development version)","text":"Brier score survival data added brier_survival(). Integrated Brier score survival data added brier_survival_integrated(). Concordance index survival data added concordance_survival(). Time-Dependent ROC curves estimation right-censored data can now calculated roc_curve_survival(). Time-Dependent ROC AUC estimation right-censored data can now calculated roc_auc_survival(). metric_set() can now used combination dynamic static survival metrics. demographic_parity(), equalized_odds(), equal_opportunity() new metrics measuring model fairness. implemented new_groupwise_metric() constructor, general interface defining group-aware metrics allows quickly flexibly defining fairness metrics problem context mind. warnings errors updated use cli package increased clarity consistency. (#456, #457, #458)","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-120","dir":"Changelog","previous_headings":"","what":"yardstick 1.2.0","title":"yardstick 1.2.0","text":"CRAN release: 2023-04-21","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-1-2-0","dir":"Changelog","previous_headings":"","what":"New Metrics","title":"yardstick 1.2.0","text":"Brier score classification added brier_class() (#139).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"improvements-1-2-0","dir":"Changelog","previous_headings":"","what":"Improvements","title":"yardstick 1.2.0","text":"global option, yardstick.event_first, hard deprecated favor using explicit argument, event_level. Setting option now produce warning, won’t effect. (#173) Removed start-message event_level argument. yardstick metric functions now use pure tidyselect interface truth, estimate, ... class probability metrics (#322). Changed default aspect ratio PR curves 1.0. Error messages now show user-facing function called (#348). classification probability metrics now fully support class_pred objects {probably} package (#341).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-1-2-0","dir":"Changelog","previous_headings":"","what":"Bug Fixes","title":"yardstick 1.2.0","text":"Using metric_set() metric created metric_tweak() longer produces error, favor arguments set metric_tweak() (#351). Metric summarizers longer error column names data conflict argument names (#382). conf_mat() longer throw errors listed internal (#327).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"developer-1-2-0","dir":"Changelog","previous_headings":"","what":"Developer","title":"yardstick 1.2.0","text":"metric_vec_template() soft deprecated favor manual flexible metric creation approach. yardstick_remove_missing() yardstick_any_missing() added treatment missing values. check_class_metric(), check_numeric_metric(), check_prob_metric() added perform standardized input checking classification, regression class probability metrics respectively. changes mean developer’s responsibility perform validation truth estimate input. (#337). metric_summarizer() soft deprecated favor specific newly added class_metric_summarizer(), numeric_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer() (#322). dots_to_estimate() soft deprecated along metric_summarizer(). dots_to_estimate() needed prob_metric_summarizer(), curve_metric_summarizer() (#329).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-110","dir":"Changelog","previous_headings":"","what":"yardstick 1.1.0","title":"yardstick 1.1.0","text":"CRAN release: 2022-09-07 Emil Hvitfeldt now maintainer (#315). Improved chained error thrown metric_set() one metric computations fails (#313).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-100","dir":"Changelog","previous_headings":"","what":"yardstick 1.0.0","title":"yardstick 1.0.0","text":"CRAN release: 2022-06-06 yardstick metrics now support case weights new case_weights argument. also includes metric-adjacent functions like roc_curve(), pr_curve(), conf_mat(), metric_set(). options argument roc_curve(), roc_auc(), roc_aunp(), roc_aunu(), metrics() passed along pROC package now deprecated longer affect. result changing ROC curve implementation supports case weights, support previous options. need options, suggest wrapping pROC custom metric (#296). conf_mat() now ignores inputs passed ... warns try thing. Previously, passed base::table(), addition case weight support, table() longer used (#295). Fixed small mistake ccc() unbiased covariance wasn’t used bias = FALSE. j_index() now throws correct warning 0 denominator computing sens() internally. Additionally, multiclass case now removes levels occurs multiclass weighted average computation, consistent metrics updated handle #118 (#265). Improved possible ambiguity documentation data argument metrics (#255). purrr removed Suggests. pROC package removed dependency (#300). Moved Custom Metrics vignette tidymodels.org (#236).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-009","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.9","title":"yardstick 0.0.9","text":"CRAN release: 2021-11-22 New metric poisson_log_loss() added (#146). sensitivity() specificity() now work correctly tune workflowsets packages (#232). roc_curve() now throws informative error truth doesn’t control event observations. dplyr 1.0.0 now required. allowed us remove multiple usages dplyr::() favor dplyr::summarise(), can now return packed data frame columns multiple rows per group. Removed internal hardcoding \"dplyr_error\" avoid issues upcoming dplyr 1.0.8 release (#244). Updated test suite testthat 3e (#243). Internal upkeep done move rlang::warn(.subclass = ) rlang::warn(class = ), since .subclass argument deprecated (#225).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-008","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.8","title":"yardstick 0.0.8","text":"CRAN release: 2021-03-28 New metric_tweak() adjusting default values optional arguments existing yardstick metric. useful quickly adjust defaults metric included metric_set(), especially metric set going used tuning tune package (#206, #182). New classification_cost() metric computing cost poor class probability prediction using user-defined costs (#3). New msd() computing mean signed deviation (also called mean signed difference, mean signed error) (#183, @datenzauberai). class_pred objects probably package now supported, automatically converted factors computing metric. Note means equivocal values materialized NA (#198). kap() metric new weighting argument apply linear quadratic weightings computing kappa value (#2, #125, @jonthegeek). sens() undefined computing ppv(), npv(), j_index(), bal_accuracy(), sensitivity warning now correctly thrown, rather recall warning (#101). autoplot() method gain curves now plots curve line top shaded polygon, resulting sharper look line (#192, @eddjberry). autoplot() methods conf_mat now respect user-defined dimension names added conf_mat(dnn = ) converting table dimension names conf_mat (#191). Added as_tibble() method metric_set objects. Printing metric_set now uses print tibble rather data frame (#186). Re-licensed package GPL-2 MIT. See consent copyright holders (#204).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-007","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.7","title":"yardstick 0.0.7","text":"CRAN release: 2020-07-13 global option, yardstick.event_first, deprecated favor new explicit argument, event_level. metric functions previously supported changing “event” level gained new argument. global option historical design decision can classified case hidden argument. Existing code relied global option continue work version yardstick, however now get -per-session warning requests update instead use explicit event_level argument. global option completely removed future version. update, follow guide (#163). roc_auc() Hand-Till multiclass estimator now ignore levels truth occur zero times actual data. methods multiclass averaging, usually returns NA, however, ignoring levels manner consistent implementations HandTill2001 pROC packages (#123). roc_auc() roc_curve() now set direction = \"<\" computing ROC curve using pROC::roc(). Results computed incorrectly direction = \"auto\" probability values predicting wrong class (#123). mn_log_loss() now respects (deprecated) global option yardstick.event_first. However, instead change relevant event level event_level argument. metric_set() now strips package name auto-labeling functions (@rorynolan, #151). three new helper functions easily creating custom metric functions: new_class_metric(), new_prob_metric(), new_numeric_metric(). Rcpp removed direct dependency.","code":"`options(yardstick.event_first = TRUE)` -> `event_level = \"first\"` (the default) `options(yardstick.event_first = FALSE)` -> `event_level = \"second\"`"},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-006","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.6","title":"yardstick 0.0.6","text":"CRAN release: 2020-03-17 roc_auc() now warns events controls provided truth column, returns NA (@dpastling, #132). Adds sensitivity() specificity() aliases sens() spec() respectively, avoids conflict packages e.g. readr::spec(). roc_aunu() roc_aunp() two new ROC AUC metrics multiclass classifiers. measure AUC class rest, roc_aunu() using uniform class distribution (#69) roc_aunp() using priori class distribution (#70).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-005","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.5","title":"yardstick 0.0.5","text":"CRAN release: 2020-01-23","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-5","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.5","text":"autoplot() heat map confusion matrices now places predicted values x axis truth values y axis consistent confusion matrix print() method. autoplot() mosaic plot confusion matrices x y axis labels backwards. corrected.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-004","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.4","title":"yardstick 0.0.4","text":"CRAN release: 2019-08-26","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-4","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.4","text":"iic() new numeric metric computing index ideality correlation. can seen potential alternative traditional correlation coefficient, used QSAR models (@jyuu, #115). average_precision() new probability metric can used alternative pr_auc(). benefit avoiding issues ambiguity case recall == 0 current number false positives 0.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-4","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.4","text":"metric_set() output now includes metrics attribute contains list original metric functions used generate metric set. metric function now direction attribute attached , specifying whether minimize maximize metric. Classification metrics can potentially 0 value denominator now throw informative warning case occurs. include recall(), precision(), sens(), spec() (#98). autoplot() method pr_curve() improved always set axis limits c(0, 1). valid arguments pROC::roc() now utilized, including passed pROC::auc(). Documentation class probability metrics improved informative examples (@rudeboybert, #100).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-0-0-4","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"yardstick 0.0.4","text":"mn_log_loss() now uses min/max rule computing log estimated probabilities avoid problematic undefined log values (#103). pr_curve() now places 1 first precision value, rather NA. NA technically correct precision undefined , 1 practically correct generates correct PR Curve graph , importantly, allows pr_auc() compute correct AUC. pr_curve() generate wrong results somewhat rare case two class probability estimates , different truth values. pr_curve() (subsequently pr_auc()) now generates correct curve duplicate class probability values (reported @dariyasydykova, #93). Binary mcc() now avoids integer overflow confusion matrix elements large (#108).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-003","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.3","title":"yardstick 0.0.3","text":"CRAN release: 2019-03-08","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-3","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.3","text":"mase() numeric metric mean absolute scaled error. generally useful forecasting time series (@alexhallam, #68). huber_loss() numeric metric less sensitive outliers rmse(), sensitive mae() small errors (@blairj09, #71). huber_loss_pseudo() smoothed form huber_loss() (@blairj09, #71). smape() numeric metric based percentage errors (@riazhedayati, #67). conf_mat objects now two ggplot2::autoplot() methods easy visualization confusion matrix either heat map mosaic plot (@EmilHvitfeldt, #10).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-3","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.3","text":"metric_set() now returns classed function. numeric metrics used, \"numeric_metric_set\" function returned. class probability metrics used, \"class_prob_metric_set\" returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-0-0-3","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"yardstick 0.0.3","text":"Tests related fixed R 3.6 sample() function fixed. f_meas() propagates NA values precision() recall() correctly (#77). \"micro\" estimators now propagate NA values correctly. roc_auc(estimator = \"hand_till\") now correctly computes metric column names probability matrix exact levels truth. Note computation still assumes order supplied probability matrix columns still matches order levels(truth), like multiclass metrics (#86).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-002","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.2","title":"yardstick 0.0.2","text":"CRAN release: 2018-11-05","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"breaking-changes-0-0-2","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"yardstick 0.0.2","text":"desire standardize yardstick API drove breaking changes. output metric now line tidy principles, returning tibble rather single numeric. Additionally, metrics now standard argument list able switch metrics combine together effortlessly. metrics now return tibble rather single numeric value. format allows metrics work grouped data frames (resamples). also allows bundle multiple metrics together new function, metric_set(). class probability metrics, now 1 column can passed ... binary implementation used. metrics longer select first column multiple columns supplied, instead throw error. summary() method conf_mat objects now returns tibble consistent change metric functions. naming consistency, mnLogLoss() renamed mn_log_loss() mn_log_loss() now returns negative log loss multinomial distribution. argument na.rm changed na_rm metrics align tidymodels model implementation principles.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"core-features-0-0-2","dir":"Changelog","previous_headings":"","what":"Core features","title":"yardstick 0.0.2","text":"metric now vector interface go alongside data frame interface. vector functions end _vec(). vector interface accepts vector/matrix inputs returns single numeric value. Multiclass support added classification metric. support varies one metric next, generally macro micro averaging available metrics, metrics specialized multiclass implementations (example, roc_auc() supports multiclass generalization presented paper Hand Till). information, see vignette(\"multiclass\", \"yardstick\"). metrics now work grouped data frames. produces tibble many rows groups, useful used alongside resampling techniques.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-2","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.2","text":"mape() calculates mean absolute percent error. kap() metric similar accuracy() calculates Cohen’s kappa. detection_prevalence() calculates number predicted positive events relative total number predictions. bal_accuracy() calculates balanced accuracy average sensitivity specificity. roc_curve() calculates receiver operator curves returns results tibble. pr_curve() calculates precision recall curves. gain_curve() lift_curve() calculate information used gain lift curves. gain_capture() measure performance similar spirit AUC applied gain curve. pr_curve(), roc_curve(), gain_curve() lift_curve() ggplot2::autoplot() methods easy visualization. metric_set() constructs functions calculate multiple metrics .","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-2","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.2","text":"infrastructure creating metrics exposed allow users extend yardstick work metrics. might want want metrics work grouped data frames box, want standardization error checking yardstick already provides. See vignette(\"custom-metrics\", \"yardstick\") examples. vignette describing three classes metrics used yardstick added. also includes list every metric available, grouped class. See vignette(\"metric-types\", \"yardstick\"). error messages yardstick now much informative, better feedback types input metric can use kinds metrics can used together (.e. metric_set()). now grouped_df method conf_mat() returns tibble list column conf_mat objects. metric now help page. allows us better document nuances metric without cluttering help pages metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"dependencies-0-0-2","dir":"Changelog","previous_headings":"","what":"Dependencies","title":"yardstick 0.0.2","text":"broom removed Depends, replaced generics Suggests. tidyr ggplot2 moved Suggests. MLmetrics removed dependency.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-001","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.1","title":"yardstick 0.0.1","text":"CRAN release: 2017-11-12 First CRAN release","code":""}] +[{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"our-pledge","dir":"","previous_headings":"","what":"Our Pledge","title":"Contributor Covenant Code of Conduct","text":"members, contributors, leaders pledge make participation community harassment-free experience everyone, regardless age, body size, visible invisible disability, ethnicity, sex characteristics, gender identity expression, level experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, sexual identity orientation. pledge act interact ways contribute open, welcoming, diverse, inclusive, healthy community.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"our-standards","dir":"","previous_headings":"","what":"Our Standards","title":"Contributor Covenant Code of Conduct","text":"Examples behavior contributes positive environment community include: Demonstrating empathy kindness toward people respectful differing opinions, viewpoints, experiences Giving gracefully accepting constructive feedback Accepting responsibility apologizing affected mistakes, learning experience Focusing best just us individuals, overall community Examples unacceptable behavior include: use sexualized language imagery, sexual attention advances kind Trolling, insulting derogatory comments, personal political attacks Public private harassment Publishing others’ private information, physical email address, without explicit permission conduct reasonably considered inappropriate professional setting","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"enforcement-responsibilities","dir":"","previous_headings":"","what":"Enforcement Responsibilities","title":"Contributor Covenant Code of Conduct","text":"Community leaders responsible clarifying enforcing standards acceptable behavior take appropriate fair corrective action response behavior deem inappropriate, threatening, offensive, harmful. Community leaders right responsibility remove, edit, reject comments, commits, code, wiki edits, issues, contributions aligned Code Conduct, communicate reasons moderation decisions appropriate.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"scope","dir":"","previous_headings":"","what":"Scope","title":"Contributor Covenant Code of Conduct","text":"Code Conduct applies within community spaces, also applies individual officially representing community public spaces. Examples representing community include using official e-mail address, posting via official social media account, acting appointed representative online offline event.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"enforcement","dir":"","previous_headings":"","what":"Enforcement","title":"Contributor Covenant Code of Conduct","text":"Instances abusive, harassing, otherwise unacceptable behavior may reported community leaders responsible enforcement codeofconduct@posit.co. complaints reviewed investigated promptly fairly. community leaders obligated respect privacy security reporter incident.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"enforcement-guidelines","dir":"","previous_headings":"","what":"Enforcement Guidelines","title":"Contributor Covenant Code of Conduct","text":"Community leaders follow Community Impact Guidelines determining consequences action deem violation Code Conduct:","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"id_1-correction","dir":"","previous_headings":"Enforcement Guidelines","what":"1. Correction","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Use inappropriate language behavior deemed unprofessional unwelcome community. Consequence: private, written warning community leaders, providing clarity around nature violation explanation behavior inappropriate. public apology may requested.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"id_2-warning","dir":"","previous_headings":"Enforcement Guidelines","what":"2. Warning","title":"Contributor Covenant Code of Conduct","text":"Community Impact: violation single incident series actions. Consequence: warning consequences continued behavior. interaction people involved, including unsolicited interaction enforcing Code Conduct, specified period time. includes avoiding interactions community spaces well external channels like social media. Violating terms may lead temporary permanent ban.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"id_3-temporary-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"3. Temporary Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: serious violation community standards, including sustained inappropriate behavior. Consequence: temporary ban sort interaction public communication community specified period time. public private interaction people involved, including unsolicited interaction enforcing Code Conduct, allowed period. Violating terms may lead permanent ban.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"id_4-permanent-ban","dir":"","previous_headings":"Enforcement Guidelines","what":"4. Permanent Ban","title":"Contributor Covenant Code of Conduct","text":"Community Impact: Demonstrating pattern violation community standards, including sustained inappropriate behavior, harassment individual, aggression toward disparagement classes individuals. Consequence: permanent ban sort public interaction within community.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CODE_OF_CONDUCT.html","id":"attribution","dir":"","previous_headings":"","what":"Attribution","title":"Contributor Covenant Code of Conduct","text":"Code Conduct adapted Contributor Covenant, version 2.1, available https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. Community Impact Guidelines inspired [Mozilla’s code conduct enforcement ladder][https://github.com/mozilla/inclusion]. answers common questions code conduct, see FAQ https://www.contributor-covenant.org/faq. Translations available https://www.contributor-covenant.org/translations.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CONTRIBUTING.html","id":null,"dir":"","previous_headings":"","what":"Contributing to tidymodels","title":"Contributing to tidymodels","text":"detailed information contributing tidymodels packages, see development contributing guide.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CONTRIBUTING.html","id":"documentation","dir":"","previous_headings":"","what":"Documentation","title":"Contributing to tidymodels","text":"Typos grammatical errors documentation may edited directly using GitHub web interface, long changes made source file. YES ✅: edit roxygen comment .R file R/ directory. 🚫: edit .Rd file man/ directory. use roxygen2, Markdown syntax, documentation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CONTRIBUTING.html","id":"code","dir":"","previous_headings":"","what":"Code","title":"Contributing to tidymodels","text":"submit 🎯 pull request tidymodels package, always file issue confirm tidymodels team agrees idea happy basic proposal. tidymodels packages work together. package contains unit tests, integration tests tests using packages contained extratests. pull requests, recommend create fork repo usethis::create_from_github(), initiate new branch usethis::pr_init(). Look build status making changes. README contains badges continuous integration services used package. New code follow tidyverse style guide. can use styler package apply styles, please don’t restyle code nothing PR. user-facing changes, add bullet top NEWS.md current development version header describing changes made followed GitHub username, links relevant issue(s)/PR(s). use testthat. Contributions test cases included easier accept. contribution spans use one package, consider building extratests changes check breakages /adding new tests . Let us know PR ran extra tests. yardstick package, test objects created via helper functions tests/testthat/ folder.","code":""},{"path":"https://yardstick.tidymodels.org/dev/CONTRIBUTING.html","id":"code-of-conduct","dir":"","previous_headings":"Code","what":"Code of Conduct","title":"Contributing to tidymodels","text":"project released Contributor Code Conduct. contributing project, agree abide terms.","code":""},{"path":"https://yardstick.tidymodels.org/dev/LICENSE.html","id":null,"dir":"","previous_headings":"","what":"MIT License","title":"MIT License","text":"Copyright (c) 2023 yardstick authors Permission hereby granted, free charge, person obtaining copy software associated documentation files (“Software”), deal Software without restriction, including without limitation rights use, copy, modify, merge, publish, distribute, sublicense, /sell copies Software, permit persons Software furnished , subject following conditions: copyright notice permission notice shall included copies substantial portions Software. SOFTWARE PROVIDED “”, WITHOUT WARRANTY KIND, EXPRESS IMPLIED, INCLUDING LIMITED WARRANTIES MERCHANTABILITY, FITNESS PARTICULAR PURPOSE NONINFRINGEMENT. EVENT SHALL AUTHORS COPYRIGHT HOLDERS LIABLE CLAIM, DAMAGES LIABILITY, WHETHER ACTION CONTRACT, TORT OTHERWISE, ARISING , CONNECTION SOFTWARE USE DEALINGS SOFTWARE.","code":""},{"path":"https://yardstick.tidymodels.org/dev/MAINTENANCE.html","id":"current-state","dir":"","previous_headings":"","what":"Current state","title":"NA","text":"yardstick reached 1.0.0 status stable. majority users, 3 types metrics, internal class defined new_metric(): Numeric metrics Class metrics Class probability metrics yardstick bit unique actual functions exports, like accuracy(), extra classes attributes attached . allows used metric_set(), decide whether two metric functions allowed combined metric set . example, two numeric metric functions can combined, can’t combine numeric metric class metric. exception can combine class metric class probability metric - resulting function get back metric_set(), class metric use estimate interface class probability metric use … one. current public facing API, don’t see major changes needing made. ’m fairly happy 3 core metric classes work. think work yardstick done improving internal helpers (see Known issues), extending yardstick new metric class types (see Future directions). likely internal helpers improved first can add new metric class types, quite complex , making extending yardstick fairly difficult.","code":""},{"path":"https://yardstick.tidymodels.org/dev/MAINTENANCE.html","id":"known-issues","dir":"","previous_headings":"","what":"Known issues","title":"NA","text":"similar problem metric_vec_template(). currently tries handle validation function calling different types metrics. makes extremely complex, hard extend, probably bit brittle. particular validate_truth_estimate_checks() fairly complex S3 dispatch perform validation (kind home grown double dispatch truth estimate) might able rewritten cleaner way separate metric_vec_template() functions different kinds metric types. issues high cognitive overhead comes play, making hard add features: https://github.com/tidymodels/yardstick/issues/311 https://github.com/tidymodels/yardstick/issues/305 https://github.com/tidymodels/yardstick/issues/251 complexity validate_truth_estimate_checks() reduced instead creating check_*() helpers force metric writers call . provide useful ones, just call metric_vec() function ’d avoid double dispatch altogether ’d charge calling correct check_*() function based type truth estimate metric works . Something like check_factor_truth_factor_estimate(truth, estimate). probably help #305.","code":""},{"path":"https://yardstick.tidymodels.org/dev/MAINTENANCE.html","id":"future-directions","dir":"","previous_headings":"","what":"Future directions","title":"NA","text":"’d like officially hard deprecate support yardstick.event_first, global option soft deprecated awhile favor explicit event_level argument. definitely run revdeps removing , fairly aggressive getting rid point. can probably remove next minor release. https://github.com/tidymodels/yardstick/issues/173 number people seem interested calibration metrics calibration curves. might combined probably package way https://github.com/tidymodels/yardstick/issues/150. Fairness metrics seem fairly popular might fit yardstick, don’t clear sense ’d implemented. https://github.com/tidymodels/yardstick/issues/176 know want add survival metrics yardstick point. POC pull request adds basic infrastructure, isn’t fully fleshed https://github.com/tidymodels/yardstick/pull/222. Adding yardstick bit challenging reasons: metrics actually care adding? isn’t entirely clear list metrics really care . think critical scope survival support narrowly, example, PR supports right censoring right now. Trying add everything hard. API look like? Right now, well-defined API yardstick functions work. take truth estimate vectors, columns data frame. probability metrics take … instead single estimate column need supply multiple estimate columns. survival metrics, things bit complex consider censorship aspect. PR linked requires truth Surv object, censorship baked . estimate column, PR requires list-tibbles (length truth), .e. output predict() function censored. makes API rather complex normal usage, think goal align nicely usage within tidymodels. metric PR dummy/naive roc auc curve survival analysis. imagine “real” survival metrics might need take kind time argument, probably required argument, might make API little tricky use tune metric_set()s. metric PR also adds new .time column output metric. yardstick functions . special nature function signature survival metrics, extra .time column output, probably can’t combined non-survival metrics metric_set(). reason, survival metric functions get class, .e. new_metric(), metric_set() use allow metrics class combined together. think _vec() variant PR little strange. _vec() variants return single numeric value, one return data frame. might just function work, might also consider exporting _vec() variant survival metrics. thing remember adding new truth types like Surv validation estimator helpers gain new S3 methods handle . example finalize_estimator_default() validate_truth_estimate_types() internal generics needed new S3 methods handle Surv (PR).","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/grouping.html","id":"group-awareness","dir":"Articles","previous_headings":"","what":"Group-awareness","title":"Grouping behavior in yardstick","text":"Even implementation groupwise metrics, yardstick metrics group-aware. grouped data passed group-aware metric, return metric values calculated group. demonstrate, ’ll make use hpc_cv data set, containing class probabilities class predictions linear discriminant analysis fit HPC data set Kuhn Johnson (2013). model evaluated via 10-fold cross-validation, predictions folds included. purposes vignette, ’ll also add column batch data select columns class probabilities, don’t need. wanted compute accuracy first resampled model, write: metric function returns one row, giving .metric, .estimator, .estimate whole data set passed. instead group data fold, metric functions like accuracy know compute values group; output, row correspond Resample. Note first row, corresponding Fold01, gives value manually filtering observations corresponding first resample computing accuracy. behavior mean group-awareness. grouped data passed group-aware metric functions, return values group.","code":"tibble(hpc_cv) #> # A tibble: 3,467 × 7 #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.929 0.0653 0.00579 0.0000156 Fold01 #> 5 VF VF 0.942 0.0543 0.00381 0.00000729 Fold01 #> 6 VF VF 0.951 0.0462 0.00272 0.00000384 Fold01 #> 7 VF VF 0.914 0.0782 0.00767 0.0000354 Fold01 #> 8 VF VF 0.918 0.0744 0.00726 0.0000157 Fold01 #> 9 VF VF 0.843 0.128 0.0296 0.000192 Fold01 #> 10 VF VF 0.920 0.0728 0.00703 0.0000147 Fold01 #> # ℹ 3,457 more rows set.seed(1) hpc <- tibble(hpc_cv) %>% mutate(batch = sample(c(\"a\", \"b\"), nrow(.), replace = TRUE)) %>% select(-c(VF, F, M, L)) hpc #> # A tibble: 3,467 × 4 #> obs pred Resample batch #> #> 1 VF VF Fold01 a #> 2 VF VF Fold01 b #> 3 VF VF Fold01 a #> 4 VF VF Fold01 a #> 5 VF VF Fold01 b #> 6 VF VF Fold01 a #> 7 VF VF Fold01 a #> 8 VF VF Fold01 a #> 9 VF VF Fold01 b #> 10 VF VF Fold01 b #> # ℹ 3,457 more rows hpc %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 hpc %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699"},{"path":"https://yardstick.tidymodels.org/dev/articles/grouping.html","id":"groupwise-metrics","dir":"Articles","previous_headings":"","what":"Groupwise metrics","title":"Grouping behavior in yardstick","text":"Groupwise metrics associated data-column , passed data column, metric temporarily group column, compute values groups defined column, aggregate values computed temporary grouping back level input data’s grouping. concretely, let’s turn example pre-existing grouping data. Consider portion HPC data pertaining first resample: Suppose batches data represent two groups model performance differ. quantify degree model performance differs two groups, compute accuracy values either group separately, take difference. First, computing accuracies: Now, taking difference: Groupwise metrics encode group_by() aggregation step (case, subtraction) shown yardstick metric. can define new groupwise metric new_groupwise_metric() function: fn argument yardstick metric computed data group. name argument gives name new metric ’ve created; ’ll call “accuracy difference.” aggregate argument function defining go fn output group single numeric value. output, accuracy_diff, function subclass called metric_factory: accuracy_diff now knows take accuracy values group return difference accuracy first second result output. last thing need associate object name grouping variable pass group_by(); can pass variable name accuracy_diff : output, accuracy_diff_by_batch, yardstick metric function like : can use accuracy_diff_by_batch() metric way use accuracy(). : can also add accuracy_diff_by_batch() metric sets: Groupwise metrics group-aware. passed data grouping variables column passed first argument accuracy_diff()—case, group—accuracy_diff_by_batch() behave like yardstick metric. example: Groupwise metrics form backend fairness metrics tidymodels. learn groupwise metrics applications fairness problems, see new_groupwise_metric().","code":"hpc %>% filter(Resample == \"Fold01\") #> # A tibble: 347 × 4 #> obs pred Resample batch #> #> 1 VF VF Fold01 a #> 2 VF VF Fold01 b #> 3 VF VF Fold01 a #> 4 VF VF Fold01 a #> 5 VF VF Fold01 b #> 6 VF VF Fold01 a #> 7 VF VF Fold01 a #> 8 VF VF Fold01 a #> 9 VF VF Fold01 b #> 10 VF VF Fold01 b #> # ℹ 337 more rows acc_by_group <- hpc %>% filter(Resample == \"Fold01\") %>% group_by(batch) %>% accuracy(obs, pred) acc_by_group #> # A tibble: 2 × 4 #> batch .metric .estimator .estimate #> #> 1 a accuracy multiclass 0.713 #> 2 b accuracy multiclass 0.739 diff(c(acc_by_group$.estimate[2], acc_by_group$.estimate[1])) #> [1] -0.02518607 accuracy_diff <- new_groupwise_metric( fn = accuracy, name = \"accuracy_diff\", aggregate = function(acc_by_group) { diff(c(acc_by_group$.estimate[2], acc_by_group$.estimate[1])) } ) class(accuracy_diff) #> [1] \"metric_factory\" \"function\" accuracy_diff_by_batch <- accuracy_diff(batch) class(accuracy) #> [1] \"class_metric\" \"metric\" \"function\" class(accuracy_diff_by_batch) #> [1] \"class_metric\" \"metric\" \"function\" hpc %>% filter(Resample == \"Fold01\") %>% accuracy_diff_by_batch(obs, pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 accuracy_diff batch multiclass -0.0252 acc_ms <- metric_set(accuracy, accuracy_diff_by_batch) hpc %>% filter(Resample == \"Fold01\") %>% acc_ms(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 accuracy multiclass 0.726 NA #> 2 accuracy_diff multiclass -0.0252 batch hpc %>% group_by(Resample) %>% accuracy_diff_by_batch(obs, pred) #> # A tibble: 10 × 5 #> Resample .metric .by .estimator .estimate #> #> 1 Fold01 accuracy_diff batch multiclass -0.0252 #> 2 Fold02 accuracy_diff batch multiclass 0.106 #> 3 Fold03 accuracy_diff batch multiclass 0.0220 #> 4 Fold04 accuracy_diff batch multiclass -0.000300 #> 5 Fold05 accuracy_diff batch multiclass -0.0361 #> 6 Fold06 accuracy_diff batch multiclass 0.0153 #> 7 Fold07 accuracy_diff batch multiclass -0.0323 #> 8 Fold08 accuracy_diff batch multiclass -0.0159 #> 9 Fold09 accuracy_diff batch multiclass -0.0131 #> 10 Fold10 accuracy_diff batch multiclass -0.0255"},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"metric-types","dir":"Articles","previous_headings":"","what":"Metric types","title":"Metric types","text":"three main metric types yardstick: class, class probability, numeric. type metric standardized argument syntax, metrics return kind output (tibble 3 columns). standardization allows metrics easily grouped together used grouped data frames computing multiple resamples . five types metrics, along types inputs take. Class metrics (hard predictions) truth - factor estimate - factor Class probability metrics (soft predictions) truth - factor estimate / ... - multiple numeric columns containing class probabilities Numeric metrics truth - numeric estimate - numeric Static survival metircs truth - Surv estimate - numeric dynamic survival metrics truth - Surv ... - list data.frames, containing 3 columns .eval_time, .pred_survival, .weight_censored`","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"example","dir":"Articles","previous_headings":"","what":"Example","title":"Metric types","text":"following example, hpc_cv data set used. contains class probabilities class predictions linear discriminant analysis fit HPC data set Kuhn Johnson (2013). fit 10 fold cross-validation, predictions folds included. 1 metric, 1 resample 1 metric, 10 resamples 2 metrics, 10 resamples","code":"library(yardstick) library(dplyr) data(\"hpc_cv\") hpc_cv %>% group_by(Resample) %>% slice(1:3) #> # A tibble: 30 × 7 #> # Groups: Resample [10] #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.941 0.0544 0.00441 0.0000123 Fold02 #> 5 VF VF 0.948 0.0483 0.00347 0.00000792 Fold02 #> 6 VF VF 0.958 0.0395 0.00236 0.00000310 Fold02 #> 7 VF VF 0.939 0.0556 0.00513 0.00000790 Fold03 #> 8 VF VF 0.928 0.0642 0.00777 0.0000148 Fold03 #> 9 VF VF 0.927 0.0653 0.00786 0.0000150 Fold03 #> 10 VF VF 0.949 0.0469 0.00398 0.00000935 Fold04 #> # ℹ 20 more rows hpc_cv %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 hpc_cv %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 class_metrics <- metric_set(accuracy, kap) hpc_cv %>% group_by(Resample) %>% class_metrics(obs, estimate = pred) #> # A tibble: 20 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492"},{"path":"https://yardstick.tidymodels.org/dev/articles/metric-types.html","id":"metrics","dir":"Articles","previous_headings":"","what":"Metrics","title":"Metric types","text":"table metrics available yardstick, grouped type.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"introduction","dir":"Articles","previous_headings":"","what":"Introduction","title":"Multiclass averaging","text":"Classification metrics yardstick truth estimate columns factors implemented binary multiclass case. multiclass implementations use micro, macro, macro_weighted averaging applicable, metrics specialized multiclass implementations.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"macro-averaging","dir":"Articles","previous_headings":"","what":"Macro averaging","title":"Multiclass averaging","text":"Macro averaging reduces multiclass predictions multiple sets binary predictions, calculates corresponding metric binary cases, averages results together. example, consider precision binary case. \\[ Pr = \\frac{TP}{TP + FP} \\] multiclass case, levels , B, C D, macro averaging reduces problem multiple one-vs-comparisons. truth estimate columns recoded two levels , precision calculated based recoded columns, “relevant” column. process repeated 3 levels get total 4 precision values. results averaged together. formula representation looks like . k classes: \\[ Pr_{macro} = \\frac{Pr_1 + Pr_2 + \\ldots + Pr_k}{k} = Pr_1 \\frac{1}{k} + Pr_2 \\frac{1}{k} + \\ldots + Pr_k \\frac{1}{k} \\] \\(PR_1\\) precision calculated recoding multiclass predictions just class 1 . Note macro averaging, classes get equal weight contributing portion precision value total (1/4). might realistic calculation large amount class imbalance. case, weighted macro average might make sense, weights calculated frequency class truth column. \\[ Pr_{weighted-macro} = Pr_1 \\frac{\\#Obs_1}{N} + Pr_2 \\frac{\\#Obs_2}{N} + \\ldots + Pr_k \\frac{\\#Obs_k}{N} \\]","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"micro-averaging","dir":"Articles","previous_headings":"","what":"Micro averaging","title":"Multiclass averaging","text":"Micro averaging treats entire set data aggregate result, calculates 1 metric rather k metrics get averaged together. precision, works calculating true positive results class using numerator, calculating true positive false positive results class, using denominator. \\[ Pr_{micro} = \\frac{TP_1 + TP_2 + \\ldots + TP_k}{(TP_1 + TP_2 + \\ldots + TP_k) + (FP_1 + FP_2 + \\ldots + FP_k)} \\] case, rather class equal weight, observation gets equal weight. gives classes observations power.","code":""},{"path":"https://yardstick.tidymodels.org/dev/articles/multiclass.html","id":"specialized-multiclass-implementations","dir":"Articles","previous_headings":"","what":"Specialized multiclass implementations","title":"Multiclass averaging","text":"metrics known analytical multiclass extensions, need use averaging get estimate multiclass performance. Accuracy Kappa use definitions binary counterpart, accuracy counting number correctly predicted true values total number true values, kappa linear combination two accuracy values. Matthews correlation coefficient (MCC) known multiclass generalization well, sometimes called \\(R_K\\) statistic. Refer page details. ROC AUC interesting metric intuitively makes sense perform macro averaging, computes multiclass AUC average area multiple binary ROC curves. However, loses important property ROC AUC statistic binary case insensitive class distribution. combat , multiclass metric created retains insensitivity class distribution, easy visual interpretation like macro averaging. implemented \"hand_till\" method, default metric.","code":""},{"path":"https://yardstick.tidymodels.org/dev/authors.html","id":null,"dir":"","previous_headings":"","what":"Authors","title":"Authors and Citation","text":"Max Kuhn. Author. Davis Vaughan. Author. Emil Hvitfeldt. Author, maintainer. . Copyright holder, funder.","code":""},{"path":"https://yardstick.tidymodels.org/dev/authors.html","id":"citation","dir":"","previous_headings":"","what":"Citation","title":"Authors and Citation","text":"Kuhn M, Vaughan D, Hvitfeldt E (2023). yardstick: Tidy Characterizations Model Performance. R package version 1.2.0.9001, https://yardstick.tidymodels.org, https://github.com/tidymodels/yardstick.","code":"@Manual{, title = {yardstick: Tidy Characterizations of Model Performance}, author = {Max Kuhn and Davis Vaughan and Emil Hvitfeldt}, year = {2023}, note = {R package version 1.2.0.9001, https://yardstick.tidymodels.org}, url = {https://github.com/tidymodels/yardstick}, }"},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"overview","dir":"","previous_headings":"","what":"Overview","title":"Tidy Characterizations of Model Performance","text":"yardstick package estimate well models working using tidy data principles. See package webpage information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"installation","dir":"","previous_headings":"","what":"Installation","title":"Tidy Characterizations of Model Performance","text":"install package:","code":"install.packages(\"yardstick\") # Development version: # install.packages(\"pak\") pak::pak(\"tidymodels/yardstick\")"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"two-class-metric","dir":"","previous_headings":"","what":"Two class metric","title":"Tidy Characterizations of Model Performance","text":"example, suppose create classification model predict new data set. might data looks like : can use dplyr-like syntax compute common performance characteristics model get back data frame:","code":"library(yardstick) library(dplyr) head(two_class_example) #> truth Class1 Class2 predicted #> 1 Class2 0.00359 0.996411 Class2 #> 2 Class1 0.67862 0.321379 Class1 #> 3 Class2 0.11089 0.889106 Class2 #> 4 Class1 0.73516 0.264838 Class1 #> 5 Class2 0.01624 0.983760 Class2 #> 6 Class1 0.99928 0.000725 Class1 metrics(two_class_example, truth, predicted) #> # A tibble: 2 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 # or two_class_example %>% roc_auc(truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc binary 0.939"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"multiclass-metrics","dir":"","previous_headings":"","what":"Multiclass metrics","title":"Tidy Characterizations of Model Performance","text":"classification metrics least one multiclass extension, many multiple ways calculate multiclass metrics.","code":"data(\"hpc_cv\") hpc_cv <- as_tibble(hpc_cv) hpc_cv #> # A tibble: 3,467 × 7 #> obs pred VF F M L Resample #> #> 1 VF VF 0.914 0.0779 0.00848 0.0000199 Fold01 #> 2 VF VF 0.938 0.0571 0.00482 0.0000101 Fold01 #> 3 VF VF 0.947 0.0495 0.00316 0.00000500 Fold01 #> 4 VF VF 0.929 0.0653 0.00579 0.0000156 Fold01 #> 5 VF VF 0.942 0.0543 0.00381 0.00000729 Fold01 #> 6 VF VF 0.951 0.0462 0.00272 0.00000384 Fold01 #> 7 VF VF 0.914 0.0782 0.00767 0.0000354 Fold01 #> 8 VF VF 0.918 0.0744 0.00726 0.0000157 Fold01 #> 9 VF VF 0.843 0.128 0.0296 0.000192 Fold01 #> 10 VF VF 0.920 0.0728 0.00703 0.0000147 Fold01 #> # ℹ 3,457 more rows # Macro averaged multiclass precision precision(hpc_cv, obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision macro 0.631 # Micro averaged multiclass precision precision(hpc_cv, obs, pred, estimator = \"micro\") #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision micro 0.709"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"calculating-metrics-on-resamples","dir":"","previous_headings":"","what":"Calculating metrics on resamples","title":"Tidy Characterizations of Model Performance","text":"multiple resamples model, can use metric grouped data frame calculate metric across resamples . calculates multiclass ROC AUC using method described Hand, Till (2001), across 10 resamples .","code":"hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc hand_till 0.813 #> 2 Fold02 roc_auc hand_till 0.817 #> 3 Fold03 roc_auc hand_till 0.869 #> 4 Fold04 roc_auc hand_till 0.849 #> 5 Fold05 roc_auc hand_till 0.811 #> 6 Fold06 roc_auc hand_till 0.836 #> 7 Fold07 roc_auc hand_till 0.825 #> 8 Fold08 roc_auc hand_till 0.846 #> 9 Fold09 roc_auc hand_till 0.828 #> 10 Fold10 roc_auc hand_till 0.812"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"autoplot-methods-for-easy-visualization","dir":"","previous_headings":"","what":"Autoplot methods for easy visualization","title":"Tidy Characterizations of Model Performance","text":"Curve based methods roc_curve(), pr_curve() gain_curve() ggplot2::autoplot() methods allow powerful easy visualization.","code":"library(ggplot2) hpc_cv %>% group_by(Resample) %>% roc_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/index.html","id":"contributing","dir":"","previous_headings":"","what":"Contributing","title":"Tidy Characterizations of Model Performance","text":"project released Contributor Code Conduct. contributing project, agree abide terms. questions discussions tidymodels packages, modeling, machine learning, please post RStudio Community. think encountered bug, please submit issue. Either way, learn create share reprex (minimal, reproducible example), clearly communicate code. Check details contributing guidelines tidymodels packages get help.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":null,"dir":"Reference","previous_headings":"","what":"Accuracy — accuracy","title":"Accuracy — accuracy","text":"Accuracy proportion data predicted correctly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Accuracy — accuracy","text":"","code":"accuracy(data, ...) # S3 method for data.frame accuracy(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) accuracy_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Accuracy — accuracy","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Accuracy — accuracy","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. accuracy_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Accuracy — accuracy","text":"Accuracy extends naturally multiclass scenarios. , macro micro averaging implemented.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Accuracy — accuracy","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/accuracy.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Accuracy — accuracy","text":"","code":"library(dplyr) #> #> Attaching package: ‘dplyr’ #> The following objects are masked from ‘package:stats’: #> #> filter, lag #> The following objects are masked from ‘package:base’: #> #> intersect, setdiff, setequal, union data(\"two_class_example\") data(\"hpc_cv\") # Two class accuracy(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 # Multiclass # accuracy() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 accuracy multiclass 0.726 # Groups are respected hpc_cv %>% group_by(Resample) %>% accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699"},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the precision recall curve — average_precision","title":"Area under the precision recall curve — average_precision","text":"average_precision() alternative pr_auc() avoids ambiguity value precision recall == 0 yet false positive values (say 0, others say 1, others say undefined). computes weighted average precision values returned pr_curve(), weights increase recall previous threshold. See pr_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the precision recall curve — average_precision","text":"","code":"average_precision(data, ...) # S3 method for data.frame average_precision( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) average_precision_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the precision recall curve — average_precision","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the precision recall curve — average_precision","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. average_precision_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Area under the precision recall curve — average_precision","text":"computation average precision weighted average precision values. Assuming n rows returned pr_curve(), sum 2 n, multiplying precision value p_i increase recall previous threshold, r_i - r_(-1). $$AP = \\sum (r_{} - r_{-1}) * p_i$$ summing 2 n, precision value p_1 never used. pr_curve() returns value p_1, technically undefined tp / (tp + fp) tp = 0 fp = 0. common convention use 1 p_1, metric nice property avoiding ambiguity. hand, r_1 well defined long events (p), tp / p tp = 0, r_1 = 0. p_1 defined 1, average_precision() roc_auc() values often close one another.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the precision recall curve — average_precision","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the precision recall curve — average_precision","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/average_precision.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the precision recall curve — average_precision","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. average_precision(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision binary 0.947 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% average_precision(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision macro 0.617 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% average_precision(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 average_precision macro 0.617 # Groups are respected hpc_cv %>% group_by(Resample) %>% average_precision(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 average_precision macro 0.617 #> 2 Fold02 average_precision macro 0.625 #> 3 Fold03 average_precision macro 0.699 #> 4 Fold04 average_precision macro 0.685 #> 5 Fold05 average_precision macro 0.625 #> 6 Fold06 average_precision macro 0.656 #> 7 Fold07 average_precision macro 0.617 #> 8 Fold08 average_precision macro 0.659 #> 9 Fold09 average_precision macro 0.632 #> 10 Fold10 average_precision macro 0.611 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% average_precision(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 average_precision macro_weighted 0.750 #> 2 Fold02 average_precision macro_weighted 0.745 #> 3 Fold03 average_precision macro_weighted 0.794 #> 4 Fold04 average_precision macro_weighted 0.757 #> 5 Fold05 average_precision macro_weighted 0.740 #> 6 Fold06 average_precision macro_weighted 0.747 #> 7 Fold07 average_precision macro_weighted 0.751 #> 8 Fold08 average_precision macro_weighted 0.759 #> 9 Fold09 average_precision macro_weighted 0.714 #> 10 Fold10 average_precision macro_weighted 0.742 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") average_precision_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.6173363"},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":null,"dir":"Reference","previous_headings":"","what":"Balanced accuracy — bal_accuracy","title":"Balanced accuracy — bal_accuracy","text":"Balanced accuracy computed average sens() spec().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Balanced accuracy — bal_accuracy","text":"","code":"bal_accuracy(data, ...) # S3 method for data.frame bal_accuracy( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) bal_accuracy_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Balanced accuracy — bal_accuracy","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Balanced accuracy — bal_accuracy","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. bal_accuracy_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Balanced accuracy — bal_accuracy","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Balanced accuracy — bal_accuracy","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Balanced accuracy — bal_accuracy","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/bal_accuracy.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Balanced accuracy — bal_accuracy","text":"","code":"# Two class data(\"two_class_example\") bal_accuracy(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 bal_accuracy binary 0.837 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% bal_accuracy(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 bal_accuracy macro 0.717 # Groups are respected hpc_cv %>% group_by(Resample) %>% bal_accuracy(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 bal_accuracy macro 0.717 #> 2 Fold02 bal_accuracy macro 0.711 #> 3 Fold03 bal_accuracy macro 0.767 #> 4 Fold04 bal_accuracy macro 0.724 #> 5 Fold05 bal_accuracy macro 0.715 #> 6 Fold06 bal_accuracy macro 0.707 #> 7 Fold07 bal_accuracy macro 0.699 #> 8 Fold08 bal_accuracy macro 0.734 #> 9 Fold09 bal_accuracy macro 0.717 #> 10 Fold10 bal_accuracy macro 0.706 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% bal_accuracy(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 bal_accuracy macro_weighted 0.771 #> 2 Fold02 bal_accuracy macro_weighted 0.763 #> 3 Fold03 bal_accuracy macro_weighted 0.799 #> 4 Fold04 bal_accuracy macro_weighted 0.758 #> 5 Fold05 bal_accuracy macro_weighted 0.762 #> 6 Fold06 bal_accuracy macro_weighted 0.746 #> 7 Fold07 bal_accuracy macro_weighted 0.733 #> 8 Fold08 bal_accuracy macro_weighted 0.768 #> 9 Fold09 bal_accuracy macro_weighted 0.734 #> 10 Fold10 bal_accuracy macro_weighted 0.750 # Vector version bal_accuracy_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8366167 # Making Class2 the \"relevant\" level bal_accuracy_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8366167"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":null,"dir":"Reference","previous_headings":"","what":"Brier score for classification models — brier_class","title":"Brier score for classification models — brier_class","text":"Compute Brier score classification model.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Brier score for classification models — brier_class","text":"","code":"brier_class(data, ...) # S3 method for data.frame brier_class(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_class_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Brier score for classification models — brier_class","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Brier score for classification models — brier_class","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. brier_class_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Brier score for classification models — brier_class","text":"Brier score analogous mean squared error regression models. difference binary indicator class corresponding class probability squared averaged. function uses convention Kruppa et al (2014) divides result two. Smaller values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Brier score for classification models — brier_class","text":"Brier scores can computed way number classes. , averaging types supported.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Brier score for classification models — brier_class","text":"Kruppa, J., Liu, Y., Diener, H.-C., Holste, T., Weimar, C., Koonig, . R., Ziegler, . (2014) Probability estimation machine learning methods dichotomous multicategory outcome: Applications. Biometrical Journal, 56 (4): 564-583.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Brier score for classification models — brier_class","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_class.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Brier score for classification models — brier_class","text":"","code":"# Two class data(\"two_class_example\") brier_class(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_class binary 0.106 # Multiclass library(dplyr) data(hpc_cv) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% brier_class(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_class multiclass 0.202 # Groups are respected hpc_cv %>% group_by(Resample) %>% brier_class(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 brier_class multiclass 0.202 #> 2 Fold02 brier_class multiclass 0.215 #> 3 Fold03 brier_class multiclass 0.177 #> 4 Fold04 brier_class multiclass 0.204 #> 5 Fold05 brier_class multiclass 0.213 #> 6 Fold06 brier_class multiclass 0.214 #> 7 Fold07 brier_class multiclass 0.221 #> 8 Fold08 brier_class multiclass 0.209 #> 9 Fold09 brier_class multiclass 0.235 #> 10 Fold10 brier_class multiclass 0.218"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent Brier score for right censored data — brier_survival","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"Compute time-dependent Brier score right censored data, mean squared error time point .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"","code":"brier_survival(data, ...) # S3 method for data.frame brier_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_survival_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. brier_survival_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes Brier score. account censoring, inverse probability censoring weights (IPCW) used calculations. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument. Smaller values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"E. Graf, C. Schmoor, W. Sauerbrei, M. Schumacher, “Assessment comparison prognostic classification schemes survival data,” Statistics Medicine, vol. 18, . 17-18, pp. 2529–2545, 1999.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent Brier score for right censored data — brier_survival","text":"","code":"library(dplyr) lung_surv %>% brier_survival( truth = surv_obj, .pred ) #> # A tibble: 5 × 4 #> .metric .estimator .eval_time .estimate #> #> 1 brier_survival standard 100 0.109 #> 2 brier_survival standard 200 0.194 #> 3 brier_survival standard 300 0.219 #> 4 brier_survival standard 400 0.222 #> 5 brier_survival standard 500 0.197"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":null,"dir":"Reference","previous_headings":"","what":"Integrated Brier score for right censored data — brier_survival_integrated","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"Compute integrated Brier score right censored data, overall calculation model performance values .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"","code":"brier_survival_integrated(data, ...) # S3 method for data.frame brier_survival_integrated(data, truth, ..., na_rm = TRUE, case_weights = NULL) brier_survival_integrated_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. brier_survival_integrated_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"integrated time-dependent brier score calculated \"area curve\" fashion. brier score calculated value .eval_time. area calculated via trapezoidal rule. area divided largest value .eval_time bring scale traditional brier score. Smaller values score associated better model performance. formulation takes survival probability predictions one specific evaluation times , time, computes Brier score. account censoring, inverse probability censoring weights (IPCW) used calculations. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"E. Graf, C. Schmoor, W. Sauerbrei, M. Schumacher, “Assessment comparison prognostic classification schemes survival data,” Statistics Medicine, vol. 18, . 17-18, pp. 2529–2545, 1999.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/brier_survival_integrated.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Integrated Brier score for right censored data — brier_survival_integrated","text":"","code":"library(dplyr) lung_surv %>% brier_survival_integrated( truth = surv_obj, .pred ) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 brier_survival_integrated standard 0.158"},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":null,"dir":"Reference","previous_headings":"","what":"Concordance correlation coefficient — ccc","title":"Concordance correlation coefficient — ccc","text":"Calculate concordance correlation coefficient.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Concordance correlation coefficient — ccc","text":"","code":"ccc(data, ...) # S3 method for data.frame ccc( data, truth, estimate, bias = FALSE, na_rm = TRUE, case_weights = NULL, ... ) ccc_vec(truth, estimate, bias = FALSE, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Concordance correlation coefficient — ccc","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. bias logical; biased estimate variance used (Lin (1989))? na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Concordance correlation coefficient — ccc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. ccc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Concordance correlation coefficient — ccc","text":"ccc() metric consistency/correlation accuracy, metrics rmse() strictly accuracy metrics rsq() strictly consistency/correlation","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Concordance correlation coefficient — ccc","text":"Lin, L. (1989). concordance correlation coefficient evaluate reproducibility. Biometrics, 45 (1), 255-268. Nickerson, C. (1997). note \"concordance correlation coefficient evaluate reproducibility\". Biometrics, 53(4), 1503-1507.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Concordance correlation coefficient — ccc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ccc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Concordance correlation coefficient — ccc","text":"","code":"# Supply truth and predictions as bare column names ccc(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ccc standard 0.937 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% ccc(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 ccc standard 0.935 #> 2 10 ccc standard 0.937 #> 3 2 ccc standard 0.943 #> 4 3 ccc standard 0.956 #> 5 4 ccc standard 0.944 #> 6 5 ccc standard 0.925 #> 7 6 ccc standard 0.933 #> 8 7 ccc standard 0.922 #> 9 8 ccc standard 0.955 #> 10 9 ccc standard 0.940 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.939"},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for checking inputs in new metrics — check_metric","title":"Developer function for checking inputs in new metrics — check_metric","text":"check_numeric_metric(), check_class_metric(), check_prob_metric() useful alongside metric-summarizers implementing new custom metrics. metric-summarizers call metric function inside dplyr::summarise(). functions perform checks inputs accordance type metric used.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for checking inputs in new metrics — check_metric","text":"","code":"check_numeric_metric(truth, estimate, case_weights, call = caller_env()) check_class_metric( truth, estimate, case_weights, estimator, call = caller_env() ) check_prob_metric( truth, estimate, case_weights, estimator, call = caller_env() ) check_dynamic_survival_metric( truth, estimate, case_weights, call = caller_env() ) check_static_survival_metric( truth, estimate, case_weights, call = caller_env() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/check_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for checking inputs in new metrics — check_metric","text":"truth realized vector truth. check_numeric_metric(), numeric vector. check_class_metric(), factor. check_prob_metric(), factor. check_dynamic_survival_metric(), Surv object. check_static_survival_metric(), Surv object. estimate realized estimate result. check_numeric_metric(), numeric vector. check_class_metric(), factor. check_prob_metric(), numeric vector binary truth, numeric matrix multic-class truth. check_dynamic_survival_metric(), list-column data.frames. check_static_survival_metric(), numeric vector. case_weights realized case weights, numeric vector. must length truth. call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. estimator can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":null,"dir":"Reference","previous_headings":"","what":"Costs function for poor classification — classification_cost","title":"Costs function for poor classification — classification_cost","text":"classification_cost() calculates cost poor prediction based user-defined costs. costs multiplied estimated class probabilities mean cost returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Costs function for poor classification — classification_cost","text":"","code":"classification_cost(data, ...) # S3 method for data.frame classification_cost( data, truth, ..., costs = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) classification_cost_vec( truth, estimate, costs = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Costs function for poor classification — classification_cost","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. costs data frame columns \"truth\", \"estimate\", \"cost\". \"truth\" \"estimate\" character columns containing unique combinations levels truth factor. \"costs\" numeric column representing cost applied \"estimate\" predicted, true result \"truth\". often case \"truth\" == \"estimate\", cost zero (penalty correct predictions). combinations levels truth missing, costs assumed zero. NULL, equal costs used, applying cost 0 correct predictions, cost 1 incorrect predictions. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Costs function for poor classification — classification_cost","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. class_cost_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Costs function for poor classification — classification_cost","text":"example, suppose three classes: \"\", \"B\", \"C\". Suppose truly \"\" observation class probabilities = 0.3 / B = 0.3 / C = 0.4. Suppose , true result class \"\", costs class = 0 / B = 5 / C = 10, penalizing probability incorrectly predicting \"C\" predicting \"B\". cost prediction 0.3 * 0 + 0.3 * 5 + 0.4 * 10. calculation done sample individual costs averaged.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Costs function for poor classification — classification_cost","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/classification_cost.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Costs function for poor classification — classification_cost","text":"","code":"library(dplyr) # --------------------------------------------------------------------------- # Two class example data(two_class_example) # Assuming `Class1` is our \"event\", this penalizes false positives heavily costs1 <- tribble( ~truth, ~estimate, ~cost, \"Class1\", \"Class2\", 1, \"Class2\", \"Class1\", 2 ) # Assuming `Class1` is our \"event\", this penalizes false negatives heavily costs2 <- tribble( ~truth, ~estimate, ~cost, \"Class1\", \"Class2\", 2, \"Class2\", \"Class1\", 1 ) classification_cost(two_class_example, truth, Class1, costs = costs1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost binary 0.288 classification_cost(two_class_example, truth, Class1, costs = costs2) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost binary 0.260 # --------------------------------------------------------------------------- # Multiclass data(hpc_cv) # Define cost matrix from Kuhn and Johnson (2013) hpc_costs <- tribble( ~estimate, ~truth, ~cost, \"VF\", \"VF\", 0, \"VF\", \"F\", 1, \"VF\", \"M\", 5, \"VF\", \"L\", 10, \"F\", \"VF\", 1, \"F\", \"F\", 0, \"F\", \"M\", 5, \"F\", \"L\", 5, \"M\", \"VF\", 1, \"M\", \"F\", 1, \"M\", \"M\", 0, \"M\", \"L\", 1, \"L\", \"VF\", 1, \"L\", \"F\", 1, \"L\", \"M\", 1, \"L\", \"L\", 0 ) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% classification_cost(obs, VF:L, costs = hpc_costs) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 classification_cost multiclass 0.779 # Groups are respected hpc_cv %>% group_by(Resample) %>% classification_cost(obs, VF:L, costs = hpc_costs) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 classification_cost multiclass 0.779 #> 2 Fold02 classification_cost multiclass 0.735 #> 3 Fold03 classification_cost multiclass 0.654 #> 4 Fold04 classification_cost multiclass 0.754 #> 5 Fold05 classification_cost multiclass 0.777 #> 6 Fold06 classification_cost multiclass 0.737 #> 7 Fold07 classification_cost multiclass 0.743 #> 8 Fold08 classification_cost multiclass 0.749 #> 9 Fold09 classification_cost multiclass 0.760 #> 10 Fold10 classification_cost multiclass 0.771"},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Concordance index for right-censored data — concordance_survival","title":"Concordance index for right-censored data — concordance_survival","text":"Compute Concordance index right-censored data","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Concordance index for right-censored data — concordance_survival","text":"","code":"concordance_survival(data, ...) # S3 method for data.frame concordance_survival( data, truth, estimate, na_rm = TRUE, case_weights = NULL, ... ) concordance_survival_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Concordance index for right-censored data — concordance_survival","text":"data data.frame containing columns specified truth .... ... Currently used. truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. estimate column identifier predicted time, numeric variables. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Concordance index for right-censored data — concordance_survival","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. concordance_survival_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Concordance index for right-censored data — concordance_survival","text":"concordance index defined proportion comparable pairs predictions outcomes concordant. Two observations comparable : observations experienced event (different times), observation shorter observed survival time experienced event, case event-free subject “outlived” . pair comparable experienced events time. Concordance intuitively means two samples ordered correctly model. specifically, two samples concordant, one higher estimated risk score shorter actual survival time. Larger values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Concordance index for right-censored data — concordance_survival","text":"Harrell, F.E., Califf, R.M., Pryor, D.B., Lee, K.L., Rosati, R., “Multivariable prognostic models: issues developing models, evaluating assumptions adequacy, measuring reducing errors”, Statistics Medicine, 15(4), 361-87, 1996.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Concordance index for right-censored data — concordance_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/concordance_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Concordance index for right-censored data — concordance_survival","text":"","code":"concordance_survival( data = lung_surv, truth = surv_obj, estimate = .pred_time ) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 concordance_survival standard 0.637"},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":null,"dir":"Reference","previous_headings":"","what":"Confusion Matrix for Categorical Data — conf_mat","title":"Confusion Matrix for Categorical Data — conf_mat","text":"Calculates cross-tabulation observed predicted classes.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Confusion Matrix for Categorical Data — conf_mat","text":"","code":"conf_mat(data, ...) # S3 method for data.frame conf_mat( data, truth, estimate, dnn = c(\"Prediction\", \"Truth\"), case_weights = NULL, ... ) # S3 method for conf_mat tidy(x, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Confusion Matrix for Categorical Data — conf_mat","text":"data data frame base::table(). ... used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. dnn character vector dimnames table. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. x conf_mat object.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Confusion Matrix for Categorical Data — conf_mat","text":"conf_mat() produces object class conf_mat. contains table objects. tidy.conf_mat() generates tibble columns name (cell identifier) value (cell count). used grouped data frame, conf_mat() returns tibble containing columns groups along conf_mat, list-column element conf_mat object.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Confusion Matrix for Categorical Data — conf_mat","text":"conf_mat() objects, broom tidy() method created collapses cell counts cell data frame easy manipulation. also summary() method computes various classification metrics . See summary.conf_mat() ggplot2::autoplot() method quickly visualizing matrix. heatmap mosaic type implemented. function requires factors exactly levels.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/conf_mat.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Confusion Matrix for Categorical Data — conf_mat","text":"","code":"library(dplyr) data(\"hpc_cv\") # The confusion matrix from a single assessment set (i.e. fold) cm <- hpc_cv %>% filter(Resample == \"Fold01\") %>% conf_mat(obs, pred) cm #> Truth #> Prediction VF F M L #> VF 166 33 8 1 #> F 11 71 24 7 #> M 0 3 5 3 #> L 0 1 4 10 # Now compute the average confusion matrix across all folds in # terms of the proportion of the data contained in each cell. # First get the raw cell counts per fold using the `tidy` method library(tidyr) cells_per_resample <- hpc_cv %>% group_by(Resample) %>% conf_mat(obs, pred) %>% mutate(tidied = lapply(conf_mat, tidy)) %>% unnest(tidied) # Get the totals per resample counts_per_resample <- hpc_cv %>% group_by(Resample) %>% summarize(total = n()) %>% left_join(cells_per_resample, by = \"Resample\") %>% # Compute the proportions mutate(prop = value / total) %>% group_by(name) %>% # Average summarize(prop = mean(prop)) counts_per_resample #> # A tibble: 16 × 2 #> name prop #> #> 1 cell_1_1 0.467 #> 2 cell_1_2 0.107 #> 3 cell_1_3 0.0185 #> 4 cell_1_4 0.00259 #> 5 cell_2_1 0.0407 #> 6 cell_2_2 0.187 #> 7 cell_2_3 0.0632 #> 8 cell_2_4 0.0173 #> 9 cell_3_1 0.00173 #> 10 cell_3_2 0.00692 #> 11 cell_3_3 0.0228 #> 12 cell_3_4 0.00807 #> 13 cell_4_1 0.000575 #> 14 cell_4_2 0.0104 #> 15 cell_4_3 0.0144 #> 16 cell_4_4 0.0320 # Now reshape these into a matrix mean_cmat <- matrix(counts_per_resample$prop, byrow = TRUE, ncol = 4) rownames(mean_cmat) <- levels(hpc_cv$obs) colnames(mean_cmat) <- levels(hpc_cv$obs) round(mean_cmat, 3) #> VF F M L #> VF 0.467 0.107 0.018 0.003 #> F 0.041 0.187 0.063 0.017 #> M 0.002 0.007 0.023 0.008 #> L 0.001 0.010 0.014 0.032 # The confusion matrix can quickly be visualized using autoplot() library(ggplot2) autoplot(cm, type = \"mosaic\") autoplot(cm, type = \"heatmap\")"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":null,"dir":"Reference","previous_headings":"","what":"Demographic parity — demographic_parity","title":"Demographic parity — demographic_parity","text":"Demographic parity satisfied model's predictions predicted positive rate across groups. value 0 indicates parity across groups. Note definition depend true outcome; truth argument included outputted metrics consistency. demographic_parity() calculated difference largest smallest value detection_prevalence() across groups. Demographic parity sometimes referred group fairness, disparate impact, statistical parity. See \"Measuring Disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Demographic parity — demographic_parity","text":"","code":"demographic_parity(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Demographic parity — demographic_parity","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Demographic parity — demographic_parity","text":"function outputs yardstick fairness metric function. Given grouping variable , demographic_parity() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set detection_prevalence() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Demographic parity — demographic_parity","text":"default, function takes difference range detection_prevalence() .estimates across groups. , maximum pair-wise disparity groups return value demographic_parity()'s .estimate. finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output detection_prevalence() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted demographic_parity_2() context.","code":"# the actual default `aggregate` is: diff_range <- function(x, ...) {diff(range(x$.estimate))} demographic_parity_2 <- new_groupwise_metric( fn = \\code{\\link[=detection_prevalence]{detection_prevalence()}}, name = \"demographic_parity_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Demographic parity — demographic_parity","text":"Agarwal, ., Beygelzimer, ., Dudik, M., Langford, J., & Wallach, H. (2018). \"Reductions Approach Fair Classification.\" Proceedings 35th International Conference Machine Learning, Proceedings Machine Learning Research. 80:60-69. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/demographic_parity.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Demographic parity — demographic_parity","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `demographic_parity()` by Resample m_set <- metric_set(demographic_parity(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 demographic_parity Resample macro 2.78e-17 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, demographic_parity(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 5.60e- 1 NA #> 2 demographic_parity macro 2.78e-17 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":null,"dir":"Reference","previous_headings":"","what":"Detection prevalence — detection_prevalence","title":"Detection prevalence — detection_prevalence","text":"Detection prevalence defined number predicted positive events (true positive false positive) divided total number predictions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Detection prevalence — detection_prevalence","text":"","code":"detection_prevalence(data, ...) # S3 method for data.frame detection_prevalence( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) detection_prevalence_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Detection prevalence — detection_prevalence","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Detection prevalence — detection_prevalence","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. detection_prevalence_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Detection prevalence — detection_prevalence","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Detection prevalence — detection_prevalence","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Detection prevalence — detection_prevalence","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/detection_prevalence.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Detection prevalence — detection_prevalence","text":"","code":"# Two class data(\"two_class_example\") detection_prevalence(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 detection_prevalence binary 0.554 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% detection_prevalence(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 detection_prevalence macro 0.25 # Groups are respected hpc_cv %>% group_by(Resample) %>% detection_prevalence(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 detection_prevalence macro 0.25 #> 2 Fold02 detection_prevalence macro 0.25 #> 3 Fold03 detection_prevalence macro 0.25 #> 4 Fold04 detection_prevalence macro 0.25 #> 5 Fold05 detection_prevalence macro 0.25 #> 6 Fold06 detection_prevalence macro 0.25 #> 7 Fold07 detection_prevalence macro 0.25 #> 8 Fold08 detection_prevalence macro 0.25 #> 9 Fold09 detection_prevalence macro 0.25 #> 10 Fold10 detection_prevalence macro 0.25 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% detection_prevalence(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 detection_prevalence macro_weighted 0.413 #> 2 Fold02 detection_prevalence macro_weighted 0.409 #> 3 Fold03 detection_prevalence macro_weighted 0.404 #> 4 Fold04 detection_prevalence macro_weighted 0.411 #> 5 Fold05 detection_prevalence macro_weighted 0.407 #> 6 Fold06 detection_prevalence macro_weighted 0.411 #> 7 Fold07 detection_prevalence macro_weighted 0.405 #> 8 Fold08 detection_prevalence macro_weighted 0.406 #> 9 Fold09 detection_prevalence macro_weighted 0.402 #> 10 Fold10 detection_prevalence macro_weighted 0.408 # Vector version detection_prevalence_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.554 # Making Class2 the \"relevant\" level detection_prevalence_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.446"},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer helpers — developer-helpers","title":"Developer helpers — developer-helpers","text":"Helpers used alongside check_metric, yardstick_remove_missing metric summarizers creating new metrics. See Custom performance metrics information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer helpers — developer-helpers","text":"","code":"dots_to_estimate(data, ...) get_weights(data, estimator) finalize_estimator( x, estimator = NULL, metric_class = \"default\", call = caller_env() ) finalize_estimator_internal( metric_dispatcher, x, estimator, call = caller_env() ) validate_estimator(estimator, estimator_override = NULL, call = caller_env())"},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer helpers — developer-helpers","text":"data table truth values columns predicted values rows. ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. estimator Either NULL auto-selection, single character type estimator use. x column used autoselect estimator. generally truth column, can also table metric table methods. metric_class single character name metric autoselect estimator . match method name created finalize_estimator_internal(). call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. metric_dispatcher simple dummy object class provided metric_class. created passed along . estimator_override character vector overriding default allowed estimator list c(\"binary\", \"macro\", \"micro\", \"macro_weighted\"). Set classification estimator support methods.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"dots-gt-estimate","dir":"Reference","previous_headings":"","what":"Dots -> Estimate","title":"Developer helpers — developer-helpers","text":"dots_to_estimate() useful class probability metrics take ... rather estimate argument. constructs either single name 1 input provided ... constructs quosure expression constructs matrix many columns provided .... eventually evaluated summarise() call metric-summarizers evaluate either vector matrix use underlying vector functions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"weight-calculation","dir":"Reference","previous_headings":"","what":"Weight Calculation","title":"Developer helpers — developer-helpers","text":"get_weights() accepts confusion matrix estimator type \"macro\", \"micro\", \"macro_weighted\" returns correct weights. useful creating multiclass metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"estimator-selection","dir":"Reference","previous_headings":"","what":"Estimator Selection","title":"Developer helpers — developer-helpers","text":"finalize_estimator() engine auto-selection estimator based type x. Generally x truth column. function called vector method metric. finalize_estimator_internal() S3 generic extend metric implement following estimator types: \"binary\", \"macro\", \"micro\", \"macro_weighted\". metric support , default version finalize_estimator_internal() autoselect estimator appropriately. need create method, take form: finalize_estimator_internal.metric_name. method finalize_estimator_internal() two things: estimator NULL, autoselect estimator based type x return single character estimator. estimator NULL, validate allowed estimator metric return . using default finalize_estimator_internal(), estimator selected using following heuristics: estimator NULL, validated returned immediately auto-selection needed. x : factor - \"binary\" returned 2 levels, otherwise \"macro\" returned. numeric - \"binary\" returned. table - \"binary\" returned 2 columns, otherwise \"macro\" returned. useful table methods. matrix - \"macro\" returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/developer-helpers.html","id":"estimator-validation","dir":"Reference","previous_headings":"","what":"Estimator Validation","title":"Developer helpers — developer-helpers","text":"validate_estimator() called metric specific method finalize_estimator_internal() ensures user provided estimator right format one allowed values.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":null,"dir":"Reference","previous_headings":"","what":"Equal opportunity — equal_opportunity","title":"Equal opportunity — equal_opportunity","text":"Equal opportunity satisfied model's predictions true positive false negative rates across protected groups. value 0 indicates parity across groups. equal_opportunity() calculated difference largest smallest value sens() across groups. Equal opportunity sometimes referred equality opportunity. See \"Measuring Disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Equal opportunity — equal_opportunity","text":"","code":"equal_opportunity(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Equal opportunity — equal_opportunity","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Equal opportunity — equal_opportunity","text":"function outputs yardstick fairness metric function. Given grouping variable , equal_opportunity() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set sens() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Equal opportunity — equal_opportunity","text":"default, function takes difference range sens() .estimates across groups. , maximum pair-wise disparity groups return value equal_opportunity()'s .estimate. finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output sens() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted equal_opportunity_2() context.","code":"# the actual default `aggregate` is: diff_range <- function(x, ...) {diff(range(x$.estimate))} equal_opportunity_2 <- new_groupwise_metric( fn = \\code{\\link[=sens]{sens()}}, name = \"equal_opportunity_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Equal opportunity — equal_opportunity","text":"Hardt, M., Price, E., & Srebro, N. (2016). \"Equality opportunity supervised learning\". Advances neural information processing systems, 29. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equal_opportunity.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Equal opportunity — equal_opportunity","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `equal_opportunity()` by Resample m_set <- metric_set(equal_opportunity(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 equal_opportunity Resample macro 0.103 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, equal_opportunity(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 0.560 NA #> 2 equal_opportunity macro 0.103 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":null,"dir":"Reference","previous_headings":"","what":"Equalized odds — equalized_odds","title":"Equalized odds — equalized_odds","text":"Equalized odds satisfied model's predictions false positive, true positive, false negative, true negative rates across protected groups. value 0 indicates parity across groups. default, function takes maximum difference range sens() spec() .estimates across groups. , maximum pair-wise disparity sens() spec() groups return value equalized_odds()'s .estimate. Equalized odds sometimes referred conditional procedure accuracy equality disparate mistreatment. See \"Measuring disparity\" section details implementation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Equalized odds — equalized_odds","text":"","code":"equalized_odds(by)"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Equalized odds — equalized_odds","text":"column identifier sensitive feature. unquoted column name referring column un-preprocessed data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Equalized odds — equalized_odds","text":"function outputs yardstick fairness metric function. Given grouping variable , equalized_odds() return yardstick metric function associated data-variable grouping post-processor. outputted function first generate set sens() spec() metric values group summarizing across groups using post-processing function. outputted function data frame method intended used part metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"measuring-disparity","dir":"Reference","previous_headings":"","what":"Measuring Disparity","title":"Equalized odds — equalized_odds","text":"finer control group treatment, construct context-aware fairness metric new_groupwise_metric() function passing custom aggregate function: aggregate(), x metric_set() output sens() spec() values group, ... gives additional arguments (grouping level refer \"baseline\") pass function outputted equalized_odds_2() context.","code":"# see yardstick:::max_positive_rate_diff for the actual `aggregate()` diff_range <- function(x, ...) {diff(range(x$.estimate))} equalized_odds_2 <- new_groupwise_metric( fn = metric_set(sens, spec), name = \"equalized_odds_2\", aggregate = diff_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Equalized odds — equalized_odds","text":"Agarwal, ., Beygelzimer, ., Dudik, M., Langford, J., & Wallach, H. (2018). \"Reductions Approach Fair Classification.\" Proceedings 35th International Conference Machine Learning, Proceedings Machine Learning Research. 80:60-69. Verma, S., & Rubin, J. (2018). \"Fairness definitions explained\". Proceedings international workshop software fairness (pp. 1-7). Bird, S., Dudík, M., Edgar, R., Horn, B., Lutz, R., Milan, V., ... & Walker, K. (2020). \"Fairlearn: toolkit assessing improving fairness AI\". Microsoft, Tech. Rep. MSR-TR-2020-32.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/equalized_odds.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Equalized odds — equalized_odds","text":"","code":"library(dplyr) data(hpc_cv) head(hpc_cv) #> obs pred VF F M L Resample #> 1 VF VF 0.9136340 0.07786694 0.008479147 1.991225e-05 Fold01 #> 2 VF VF 0.9380672 0.05710623 0.004816447 1.011557e-05 Fold01 #> 3 VF VF 0.9473710 0.04946767 0.003156287 4.999849e-06 Fold01 #> 4 VF VF 0.9289077 0.06528949 0.005787179 1.564496e-05 Fold01 #> 5 VF VF 0.9418764 0.05430830 0.003808013 7.294581e-06 Fold01 #> 6 VF VF 0.9510978 0.04618223 0.002716177 3.841455e-06 Fold01 # evaluate `equalized_odds()` by Resample m_set <- metric_set(equalized_odds(Resample)) # use output like any other metric set hpc_cv %>% m_set(truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 equalized_odds Resample macro 0.103 # can mix fairness metrics and regular metrics m_set_2 <- metric_set(sens, equalized_odds(Resample)) hpc_cv %>% m_set_2(truth = obs, estimate = pred) #> # A tibble: 2 × 4 #> .metric .estimator .estimate .by #> #> 1 sens macro 0.560 NA #> 2 equalized_odds macro 0.103 Resample"},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":null,"dir":"Reference","previous_headings":"","what":"F Measure — f_meas","title":"F Measure — f_meas","text":"functions calculate f_meas() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions recall() precision().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"F Measure — f_meas","text":"","code":"f_meas(data, ...) # S3 method for data.frame f_meas( data, truth, estimate, beta = 1, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) f_meas_vec( truth, estimate, beta = 1, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"F Measure — f_meas","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. beta numeric value used weight precision recall. value 1 traditionally used corresponds harmonic mean two values values weight recall beta times important precision. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"F Measure — f_meas","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. f_meas_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"F Measure — f_meas","text":"measure \"F\" combination precision recall (see ).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"F Measure — f_meas","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"F Measure — f_meas","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"F Measure — f_meas","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"F Measure — f_meas","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"F Measure — f_meas","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/f_meas.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"F Measure — f_meas","text":"","code":"# Two class data(\"two_class_example\") f_meas(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 f_meas binary 0.849 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% f_meas(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 f_meas macro 0.563 # Groups are respected hpc_cv %>% group_by(Resample) %>% f_meas(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 f_meas macro 0.563 #> 2 Fold02 f_meas macro 0.542 #> 3 Fold03 f_meas macro 0.641 #> 4 Fold04 f_meas macro 0.593 #> 5 Fold05 f_meas macro 0.570 #> 6 Fold06 f_meas macro 0.554 #> 7 Fold07 f_meas macro 0.516 #> 8 Fold08 f_meas macro 0.601 #> 9 Fold09 f_meas macro 0.555 #> 10 Fold10 f_meas macro 0.560 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% f_meas(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 f_meas macro_weighted 0.696 #> 2 Fold02 f_meas macro_weighted 0.684 #> 3 Fold03 f_meas macro_weighted 0.739 #> 4 Fold04 f_meas macro_weighted 0.689 #> 5 Fold05 f_meas macro_weighted 0.692 #> 6 Fold06 f_meas macro_weighted 0.673 #> 7 Fold07 f_meas macro_weighted 0.646 #> 8 Fold08 f_meas macro_weighted 0.701 #> 9 Fold09 f_meas macro_weighted 0.652 #> 10 Fold10 f_meas macro_weighted 0.680 # Vector version f_meas_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8485981 # Making Class2 the \"relevant\" level f_meas_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8258065"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":null,"dir":"Reference","previous_headings":"","what":"Gain capture — gain_capture","title":"Gain capture — gain_capture","text":"gain_capture() measure performance similar AUC calculation, applied gain curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Gain capture — gain_capture","text":"","code":"gain_capture(data, ...) # S3 method for data.frame gain_capture( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) gain_capture_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Gain capture — gain_capture","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Gain capture — gain_capture","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. gain_capture_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Gain capture — gain_capture","text":"gain_capture() calculates area gain curve, baseline, divides area perfect gain curve, baseline. meant represent amount potential gain \"captured\" model. gain_capture() metric identical accuracy ratio (AR), also sometimes called gini coefficient. two generally calculated cumulative accuracy profile curve, gain curve. See Engelmann reference information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Gain capture — gain_capture","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Gain capture — gain_capture","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Gain capture — gain_capture","text":"Engelmann, Bernd & Hayden, Evelyn & Tasche, Dirk (2003). \"Measuring Discriminative Power Rating Systems,\" Discussion Paper Series 2: Banking Financial Studies 2003,01, Deutsche Bundesbank.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Gain capture — gain_capture","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_capture.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Gain capture — gain_capture","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. gain_capture(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture binary 0.879 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% gain_capture(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture macro 0.743 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% gain_capture(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 gain_capture macro 0.743 # Groups are respected hpc_cv %>% group_by(Resample) %>% gain_capture(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 gain_capture macro 0.743 #> 2 Fold02 gain_capture macro 0.727 #> 3 Fold03 gain_capture macro 0.796 #> 4 Fold04 gain_capture macro 0.748 #> 5 Fold05 gain_capture macro 0.730 #> 6 Fold06 gain_capture macro 0.754 #> 7 Fold07 gain_capture macro 0.730 #> 8 Fold08 gain_capture macro 0.747 #> 9 Fold09 gain_capture macro 0.710 #> 10 Fold10 gain_capture macro 0.731 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% gain_capture(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 gain_capture macro_weighted 0.759 #> 2 Fold02 gain_capture macro_weighted 0.745 #> 3 Fold03 gain_capture macro_weighted 0.811 #> 4 Fold04 gain_capture macro_weighted 0.734 #> 5 Fold05 gain_capture macro_weighted 0.733 #> 6 Fold06 gain_capture macro_weighted 0.730 #> 7 Fold07 gain_capture macro_weighted 0.737 #> 8 Fold08 gain_capture macro_weighted 0.730 #> 9 Fold09 gain_capture macro_weighted 0.681 #> 10 Fold10 gain_capture macro_weighted 0.737 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") gain_capture_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.7428922 # --------------------------------------------------------------------------- # Visualize gain_capture() # Visually, this represents the area under the black curve, but above the # 45 degree line, divided by the area of the shaded triangle. library(ggplot2) autoplot(gain_curve(two_class_example, truth, Class1))"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Gain curve — gain_curve","title":"Gain curve — gain_curve","text":"gain_curve() constructs full gain curve returns tibble. See gain_capture() relevant area gain curve. Also see lift_curve() closely related concept.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Gain curve — gain_curve","text":"","code":"gain_curve(data, ...) # S3 method for data.frame gain_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Gain curve — gain_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Gain curve — gain_curve","text":"tibble class gain_df gain_grouped_df columns: .n index current sample. .n_events index current unique sample. Values repeated estimate values given identical indices column. .percent_tested cumulative percentage values tested. .percent_found cumulative percentage true results relative total number true results. using case_weights argument, columns weighted. makes sense frequency weights, integer weights representing number times particular observation repeated.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Gain curve — gain_curve","text":"ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples. greater area gain curve baseline, better model. Gain curves identical CAP curves (cumulative accuracy profile). See Engelmann reference information CAP curves.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"gain-and-lift-curves","dir":"Reference","previous_headings":"","what":"Gain and Lift Curves","title":"Gain curve — gain_curve","text":"motivation behind cumulative gain lift charts visual method determine effectiveness model compared results one might expect without model. example, without model, advertise random 10% customer base, might expect capture 10% total number positive responses advertised entire customer base. Given model predicts customers likely respond, hope can accurately target 10% customer base capture >10% total number positive responses. calculation construct gain curves follows: truth estimate placed descending order estimate values (estimate single column supplied ...). cumulative number samples true results relative entire number true results found. y-axis gain chart.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Gain curve — gain_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Gain curve — gain_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Gain curve — gain_curve","text":"Engelmann, Bernd & Hayden, Evelyn & Tasche, Dirk (2003). \"Measuring Discriminative Power Rating Systems,\" Discussion Paper Series 2: Banking Financial Studies 2003,01, Deutsche Bundesbank.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Gain curve — gain_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/gain_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Gain curve — gain_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. gain_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 4 #> .n .n_events .percent_tested .percent_found #> #> 1 0 0 0 0 #> 2 1 1 0.2 0.388 #> 3 2 2 0.4 0.775 #> 4 3 3 0.6 1.16 #> 5 4 4 0.8 1.55 #> 6 5 5 1 1.94 #> 7 6 6 1.2 2.33 #> 8 7 7 1.4 2.71 #> 9 8 8 1.6 3.10 #> 10 9 9 1.8 3.49 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` library(ggplot2) library(dplyr) # Use autoplot to visualize # The top left hand corner of the grey triangle is a \"perfect\" gain curve autoplot(gain_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% gain_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples # The resample with the minimum (farthest to the left) \"perfect\" value is # used to draw the shaded region hpc_cv %>% group_by(Resample) %>% gain_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":null,"dir":"Reference","previous_headings":"","what":"Multiclass Probability Predictions — hpc_cv","title":"Multiclass Probability Predictions — hpc_cv","text":"Multiclass Probability Predictions","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Multiclass Probability Predictions — hpc_cv","text":"Kuhn, M., Johnson, K. (2013) Applied Predictive Modeling, Springer","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Multiclass Probability Predictions — hpc_cv","text":"hpc_cv data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Multiclass Probability Predictions — hpc_cv","text":"data frame contains predicted classes class probabilities linear discriminant analysis model fit HPC data set Kuhn Johnson (2013). data assessment sets 10-fold cross-validation scheme. data column columns true class (obs), class prediction (pred) columns class probability (columns VF, F, M, L). Additionally, column resample indicator included.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/hpc_cv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Multiclass Probability Predictions — hpc_cv","text":"","code":"data(hpc_cv) str(hpc_cv) #> 'data.frame':\t3467 obs. of 7 variables: #> $ obs : Factor w/ 4 levels \"VF\",\"F\",\"M\",\"L\": 1 1 1 1 1 1 1 1 1 1 ... #> $ pred : Factor w/ 4 levels \"VF\",\"F\",\"M\",\"L\": 1 1 1 1 1 1 1 1 1 1 ... #> $ VF : num 0.914 0.938 0.947 0.929 0.942 ... #> $ F : num 0.0779 0.0571 0.0495 0.0653 0.0543 ... #> $ M : num 0.00848 0.00482 0.00316 0.00579 0.00381 ... #> $ L : num 1.99e-05 1.01e-05 5.00e-06 1.56e-05 7.29e-06 ... #> $ Resample: chr \"Fold01\" \"Fold01\" \"Fold01\" \"Fold01\" ... # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section in any classification function (such as `?pr_auc`) to see how # to change this. levels(hpc_cv$obs) #> [1] \"VF\" \"F\" \"M\" \"L\""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Huber loss — huber_loss","title":"Huber loss — huber_loss","text":"Calculate Huber loss, loss function used robust regression. loss function less sensitive outliers rmse(). function quadratic small residual values linear large residual values.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Huber loss — huber_loss","text":"","code":"huber_loss(data, ...) # S3 method for data.frame huber_loss( data, truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... ) huber_loss_vec( truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Huber loss — huber_loss","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. delta single numeric value. Defines boundary loss function transitions quadratic linear. Defaults 1. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Huber loss — huber_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. huber_loss_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Huber loss — huber_loss","text":"Huber, P. (1964). Robust Estimation Location Parameter. Annals Statistics, 53 (1), 73-101.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Huber loss — huber_loss","text":"James Blair","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Huber loss — huber_loss","text":"","code":"# Supply truth and predictions as bare column names huber_loss(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 huber_loss standard 0.234 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% huber_loss(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 huber_loss standard 0.215 #> 2 10 huber_loss standard 0.212 #> 3 2 huber_loss standard 0.229 #> 4 3 huber_loss standard 0.197 #> 5 4 huber_loss standard 0.249 #> 6 5 huber_loss standard 0.208 #> 7 6 huber_loss standard 0.293 #> 8 7 huber_loss standard 0.268 #> 9 8 huber_loss standard 0.190 #> 10 9 huber_loss standard 0.218 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.228"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":null,"dir":"Reference","previous_headings":"","what":"Psuedo-Huber Loss — huber_loss_pseudo","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"Calculate Pseudo-Huber Loss, smooth approximation huber_loss(). Like huber_loss(), less sensitive outliers rmse().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"","code":"huber_loss_pseudo(data, ...) # S3 method for data.frame huber_loss_pseudo( data, truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... ) huber_loss_pseudo_vec( truth, estimate, delta = 1, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. delta single numeric value. Defines boundary loss function transitions quadratic linear. Defaults 1. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. huber_loss_pseudo_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"Huber, P. (1964). Robust Estimation Location Parameter. Annals Statistics, 53 (1), 73-101. Hartley, Richard (2004). Multiple View Geometry Computer Vision. (Second Edition). Page 619.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"James Blair","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/huber_loss_pseudo.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Psuedo-Huber Loss — huber_loss_pseudo","text":"","code":"# Supply truth and predictions as bare column names huber_loss_pseudo(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 huber_loss_pseudo standard 0.199 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% huber_loss_pseudo(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 huber_loss_pseudo standard 0.185 #> 2 10 huber_loss_pseudo standard 0.179 #> 3 2 huber_loss_pseudo standard 0.196 #> 4 3 huber_loss_pseudo standard 0.168 #> 5 4 huber_loss_pseudo standard 0.212 #> 6 5 huber_loss_pseudo standard 0.177 #> 7 6 huber_loss_pseudo standard 0.246 #> 8 7 huber_loss_pseudo standard 0.227 #> 9 8 huber_loss_pseudo standard 0.161 #> 10 9 huber_loss_pseudo standard 0.188 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.194"},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":null,"dir":"Reference","previous_headings":"","what":"Index of ideality of correlation — iic","title":"Index of ideality of correlation — iic","text":"Calculate index ideality correlation. metric studied QSPR/QSAR models good criterion predictive potential models. highly dependent correlation coefficient well mean absolute error. Note application IIC useless two conditions: negative mean absolute error positive mean absolute error zero. outliers symmetric. Since outliers context dependent, please use checks validate whether restriction holds whether resulting IIC interpretative value. IIC seen alternative traditional correlation coefficient units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Index of ideality of correlation — iic","text":"","code":"iic(data, ...) # S3 method for data.frame iic(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) iic_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Index of ideality of correlation — iic","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Index of ideality of correlation — iic","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. iic_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Index of ideality of correlation — iic","text":"Toropova, . Toropov, . (2017). \"index ideality correlation. criterion predictability QSAR models skin permeability?\" Science Total Environment. 586: 466-472.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Index of ideality of correlation — iic","text":"Joyce Cahoon","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/iic.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Index of ideality of correlation — iic","text":"","code":"# Supply truth and predictions as bare column names iic(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 iic standard 0.890 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% iic(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 iic standard 0.730 #> 2 10 iic standard 0.731 #> 3 2 iic standard 0.906 #> 4 3 iic standard 0.877 #> 5 4 iic standard 0.732 #> 6 5 iic standard 0.821 #> 7 6 iic standard 0.896 #> 8 7 iic standard 0.867 #> 9 8 iic standard 0.881 #> 10 9 iic standard 0.748 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.819"},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":null,"dir":"Reference","previous_headings":"","what":"J-index — j_index","title":"J-index — j_index","text":"Youden's J statistic defined : sens() + spec() - 1 related metric Informedness, see Details section relationship.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"J-index — j_index","text":"","code":"j_index(data, ...) # S3 method for data.frame j_index( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) j_index_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"J-index — j_index","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"J-index — j_index","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. j_index_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"J-index — j_index","text":"value J-index ranges [0, 1] 1 false positives false negatives. binary version J-index equivalent binary concept Informedness. Macro-weighted J-index equivalent multiclass informedness defined Powers, David M W (2011), equation (42).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"J-index — j_index","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"J-index — j_index","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"J-index — j_index","text":"Youden, W.J. (1950). \"Index rating diagnostic tests\". Cancer. 3: 32-35. Powers, David M W (2011). \"Evaluation: Precision, Recall F-Score ROC, Informedness, Markedness Correlation\". Journal Machine Learning Technologies. 2 (1): 37-63.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"J-index — j_index","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/j_index.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"J-index — j_index","text":"","code":"# Two class data(\"two_class_example\") j_index(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 j_index binary 0.673 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% j_index(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 j_index macro 0.434 # Groups are respected hpc_cv %>% group_by(Resample) %>% j_index(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 j_index macro 0.434 #> 2 Fold02 j_index macro 0.422 #> 3 Fold03 j_index macro 0.533 #> 4 Fold04 j_index macro 0.449 #> 5 Fold05 j_index macro 0.431 #> 6 Fold06 j_index macro 0.413 #> 7 Fold07 j_index macro 0.398 #> 8 Fold08 j_index macro 0.468 #> 9 Fold09 j_index macro 0.435 #> 10 Fold10 j_index macro 0.412 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% j_index(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 j_index macro_weighted 0.542 #> 2 Fold02 j_index macro_weighted 0.527 #> 3 Fold03 j_index macro_weighted 0.597 #> 4 Fold04 j_index macro_weighted 0.515 #> 5 Fold05 j_index macro_weighted 0.524 #> 6 Fold06 j_index macro_weighted 0.492 #> 7 Fold07 j_index macro_weighted 0.466 #> 8 Fold08 j_index macro_weighted 0.535 #> 9 Fold09 j_index macro_weighted 0.468 #> 10 Fold10 j_index macro_weighted 0.501 # Vector version j_index_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.6732334 # Making Class2 the \"relevant\" level j_index_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.6732334"},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":null,"dir":"Reference","previous_headings":"","what":"Kappa — kap","title":"Kappa — kap","text":"Kappa similar measure accuracy(), normalized accuracy expected chance alone useful one classes large frequency distributions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Kappa — kap","text":"","code":"kap(data, ...) # S3 method for data.frame kap( data, truth, estimate, weighting = \"none\", na_rm = TRUE, case_weights = NULL, ... ) kap_vec( truth, estimate, weighting = \"none\", na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Kappa — kap","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. weighting weighting apply computing scores. One : \"none\", \"linear\", \"quadratic\". Linear quadratic weighting penalizes mis-predictions \"far away\" true value. Note distance judged based ordering levels truth estimate. recommended provide ordered factors truth estimate explicitly code ordering, required. binary case, 3 weightings produce value, since ever possible 1 unit away true value. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Kappa — kap","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. kap_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Kappa — kap","text":"Kappa extends naturally multiclass scenarios. , macro micro averaging implemented.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Kappa — kap","text":"Cohen, J. (1960). \"coefficient agreement nominal scales\". Educational Psychological Measurement. 20 (1): 37-46. Cohen, J. (1968). \"Weighted kappa: Nominal scale agreement provision scaled disagreement partial credit\". Psychological Bulletin. 70 (4): 213-220.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Kappa — kap","text":"Max Kuhn Jon Harmon","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/kap.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Kappa — kap","text":"","code":"library(dplyr) data(\"two_class_example\") data(\"hpc_cv\") # Two class kap(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 kap binary 0.675 # Multiclass # kap() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% kap(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 kap multiclass 0.533 # Groups are respected hpc_cv %>% group_by(Resample) %>% kap(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 kap multiclass 0.533 #> 2 Fold02 kap multiclass 0.512 #> 3 Fold03 kap multiclass 0.594 #> 4 Fold04 kap multiclass 0.511 #> 5 Fold05 kap multiclass 0.514 #> 6 Fold06 kap multiclass 0.486 #> 7 Fold07 kap multiclass 0.454 #> 8 Fold08 kap multiclass 0.531 #> 9 Fold09 kap multiclass 0.454 #> 10 Fold10 kap multiclass 0.492"},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Lift curve — lift_curve","title":"Lift curve — lift_curve","text":"lift_curve() constructs full lift curve returns tibble. See gain_curve() closely related concept.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Lift curve — lift_curve","text":"","code":"lift_curve(data, ...) # S3 method for data.frame lift_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Lift curve — lift_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Lift curve — lift_curve","text":"tibble class lift_df lift_grouped_df columns: .n index current sample. .n_events index current unique sample. Values repeated estimate values given identical indices column. .percent_tested cumulative percentage values tested. .lift First calculate cumulative percentage true results relative total number true results. divide .percent_tested. using case_weights argument, columns weighted. makes sense frequency weights, integer weights representing number times particular observation repeated.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Lift curve — lift_curve","text":"ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"gain-and-lift-curves","dir":"Reference","previous_headings":"","what":"Gain and Lift Curves","title":"Lift curve — lift_curve","text":"motivation behind cumulative gain lift charts visual method determine effectiveness model compared results one might expect without model. example, without model, advertise random 10% customer base, might expect capture 10% total number positive responses advertised entire customer base. Given model predicts customers likely respond, hope can accurately target 10% customer base capture >10% total number positive responses. calculation construct lift curves follows: truth estimate placed descending order estimate values (estimate single column supplied ...). cumulative number samples true results relative entire number true results found. cumulative % found divided cumulative % tested construct lift value. ratio represents factor improvement uninformed model. Values >1 represent valuable model. y-axis lift chart.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Lift curve — lift_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Lift curve — lift_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Lift curve — lift_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lift_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Lift curve — lift_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. lift_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 4 #> .n .n_events .percent_tested .lift #> #> 1 0 0 0 NaN #> 2 1 1 0.2 1.94 #> 3 2 2 0.4 1.94 #> 4 3 3 0.6 1.94 #> 5 4 4 0.8 1.94 #> 6 5 5 1 1.94 #> 7 6 6 1.2 1.94 #> 8 7 7 1.4 1.94 #> 9 8 8 1.6 1.94 #> 10 9 9 1.8 1.94 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` library(ggplot2) library(dplyr) # Use autoplot to visualize autoplot(lift_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% lift_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% lift_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":null,"dir":"Reference","previous_headings":"","what":"Survival Analysis Results — lung_surv","title":"Survival Analysis Results — lung_surv","text":"Survival Analysis Results","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Survival Analysis Results — lung_surv","text":"lung_surv data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Survival Analysis Results — lung_surv","text":"data contain plausible results applying predictive survival models lung data set using censored package.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/lung_surv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Survival Analysis Results — lung_surv","text":"","code":"data(lung_surv) str(lung_surv) #> tibble [228 × 3] (S3: tbl_df/tbl/data.frame) #> $ .pred :List of 228 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.819 0.597 0.407 0.264 0.164 #> .. ..$ .weight_time : num [1:5] 100 200 300 306 306 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 455 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.627 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.59 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.763 0.625 0.498 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 200 210 210 210 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.915 0.915 0.915 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.09 1.09 1.09 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.613 0.484 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.819 0.597 0.407 0.264 0.164 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.841 0.64 0.46 0.316 0.209 #> .. ..$ .weight_time : num [1:5] 100 200 300 310 310 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.837 0.631 0.449 0.305 0.2 #> .. ..$ .weight_time : num [1:5] 100 200 300 361 361 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.707 0.707 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.41 1.41 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.851 0.659 0.484 0.341 0.233 #> .. ..$ .weight_time : num [1:5] 100 200 218 218 218 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.909 0.909 0.909 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.1 1.1 1.1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7568 0.4871 0.2858 0.1563 0.0806 #> .. ..$ .weight_time : num [1:5] 100 166 166 166 166 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 170 170 170 170 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.841 0.64 0.46 0.316 0.209 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.614 0.485 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] NA NA NA NA NA #> .. ..$ .weight_time : num [1:5] 71 71 71 71 71 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 144 144 144 144 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7524 0.4799 0.2785 0.1504 0.0765 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.858 0.674 0.503 0.361 0.251 #> .. ..$ .weight_time : num [1:5] 61 61 61 61 61 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.845 0.648 0.47 0.326 0.219 #> .. ..$ .weight_time : num [1:5] 88 88 88 88 88 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 200 300 301 301 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.752 0.752 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.33 1.33 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.945 0.865 0.777 0.688 0.602 #> .. ..$ .weight_time : num [1:5] 81 81 81 81 81 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.855 0.668 0.495 0.353 0.243 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.759 0.619 0.491 0.381 #> .. ..$ .weight_time : num [1:5] 100 200 300 371 371 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.697 0.697 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.44 1.44 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.884 0.728 0.575 0.44 0.329 #> .. ..$ .weight_time : num [1:5] 100 200 300 394 394 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.663 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.51 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.895 0.751 0.607 0.478 0.367 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.897 0.755 0.613 0.484 0.374 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.6149 0.2851 0.1125 0.0392 0.0123 #> .. ..$ .weight_time : num [1:5] 100 118 118 118 118 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.851 0.659 0.484 0.341 0.233 #> .. ..$ .weight_time : num [1:5] 100 200 300 390 390 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.663 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.51 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 12 12 12 12 12 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.896 0.753 0.611 0.481 0.371 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 473 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.73 0.443 0.243 0.123 0.058 #> .. ..$ .weight_time : num [1:5] 26 26 26 26 26 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.783 0.533 0.334 0.197 0.11 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.853 0.663 0.489 0.346 0.237 #> .. ..$ .weight_time : num [1:5] 100 107 107 107 107 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7568 0.4871 0.2858 0.1563 0.0806 #> .. ..$ .weight_time : num [1:5] 53 53 53 53 53 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.85 0.657 0.481 0.338 0.23 #> .. ..$ .weight_time : num [1:5] 100 122 122 122 122 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.748 0.4727 0.2713 0.1446 0.0725 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.76 0.62 0.492 0.382 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 93 93 93 93 93 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.764 0.626 0.499 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 460 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.834 0.625 0.441 0.297 0.193 #> .. ..$ .weight_time : num [1:5] 100 153 153 153 153 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.94 0.851 0.756 0.66 0.569 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 433 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.853 0.663 0.489 0.346 0.237 #> .. ..$ .weight_time : num [1:5] 100 145 145 145 145 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.829 0.615 0.429 0.286 0.183 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.829 0.616 0.43 0.286 0.183 #> .. ..$ .weight_time : num [1:5] 95 95 95 95 95 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.882 0.723 0.568 0.433 0.321 #> .. ..$ .weight_time : num [1:5] 100 200 300 303 303 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.744 0.744 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.34 1.34 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.836 0.63 0.448 0.304 0.199 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.882 0.723 0.568 0.433 0.321 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.914 0.792 0.666 0.548 0.442 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.893 0.747 0.601 0.47 0.359 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 189 189 189 189 #> .. ..$ .pred_censored : num [1:5] 0.995 0.954 0.954 0.954 0.954 #> .. ..$ .weight_censored: num [1:5] 1.01 1.05 1.05 1.05 1.05 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 53 53 53 53 53 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.899 0.759 0.619 0.491 0.381 #> .. ..$ .weight_time : num [1:5] 100 200 246 246 246 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.845 0.845 0.845 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.18 1.18 1.18 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.842 0.642 0.463 0.319 0.212 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.895 0.751 0.607 0.477 0.366 #> .. ..$ .weight_time : num [1:5] 65 65 65 65 65 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.936 0.843 0.743 0.643 0.55 #> .. ..$ .weight_time : num [1:5] 5 5 5 5 5 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7653 0.5014 0.3005 0.1683 0.0891 #> .. ..$ .weight_time : num [1:5] 100 132 132 132 132 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.907 0.776 0.643 0.52 0.412 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.901 0.764 0.626 0.499 0.389 #> .. ..$ .weight_time : num [1:5] 100 200 300 345 345 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.717 0.717 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.39 1.39 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.619 0.434 0.29 0.187 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 444 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.858 0.673 0.502 0.36 0.25 #> .. ..$ .weight_time : num [1:5] 100 200 223 223 223 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.895 0.895 0.895 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.12 1.12 1.12 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.82 0.6 0.411 0.267 0.167 #> .. ..$ .weight_time : num [1:5] 100 175 175 175 175 #> .. ..$ .pred_censored : num [1:5] 0.995 0.978 0.978 0.978 0.978 #> .. ..$ .weight_censored: num [1:5] 1.01 1.02 1.02 1.02 1.02 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.9 0.762 0.623 0.496 0.386 #> .. ..$ .weight_time : num [1:5] 60 60 60 60 60 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.827 0.612 0.426 0.282 0.179 #> .. ..$ .weight_time : num [1:5] 100 163 163 163 163 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7413 0.4618 0.2605 0.1362 0.0668 #> .. ..$ .weight_time : num [1:5] 65 65 65 65 65 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.843 0.643 0.463 0.32 0.213 #> .. ..$ .weight_time : num [1:5] 100 200 208 208 208 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.915 0.915 0.915 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.09 1.09 1.09 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.937 0.844 0.745 0.646 0.553 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.888 0.737 0.588 0.455 0.344 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 428 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.83 0.618 0.433 0.289 0.186 #> .. ..$ .weight_time : num [1:5] 100 200 230 230 230 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.874 0.874 0.874 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.14 1.14 1.14 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.915 0.796 0.672 0.554 0.449 #> .. ..$ .weight_time : num [1:5] 100 200 300 305 305 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7274 0.4398 0.2392 0.12 0.0563 #> .. ..$ .weight_time : num [1:5] 11 11 11 11 11 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.868 0.694 0.53 0.39 0.279 #> .. ..$ .weight_time : num [1:5] 100 132 132 132 132 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.911 0.786 0.658 0.537 0.431 #> .. ..$ .weight_time : num [1:5] 100 200 226 226 226 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.874 0.874 0.874 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.14 1.14 1.14 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.749 0.604 0.474 0.363 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 426 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.64 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.56 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.944 0.862 0.773 0.682 0.595 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.908 0.78 0.649 0.527 0.419 #> .. ..$ .weight_time : num [1:5] 100 200 300 363 363 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.707 0.707 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.41 1.41 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.874 0.706 0.545 0.407 0.295 #> .. ..$ .weight_time : num [1:5] 11 11 11 11 11 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.883 0.725 0.572 0.437 0.325 #> .. ..$ .weight_time : num [1:5] 100 176 176 176 176 #> .. ..$ .pred_censored : num [1:5] 0.995 0.972 0.972 0.972 0.972 #> .. ..$ .weight_censored: num [1:5] 1.01 1.03 1.03 1.03 1.03 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.898 0.757 0.616 0.488 0.377 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.848 0.654 0.477 0.334 0.226 #> .. ..$ .weight_time : num [1:5] 95 95 95 95 95 #> .. ..$ .pred_censored : num [1:5] 0.995 0.995 0.995 0.995 0.995 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.866 0.689 0.523 0.383 0.272 #> .. ..$ .weight_time : num [1:5] 100 NA NA NA NA #> .. ..$ .pred_censored : num [1:5] 0.995 NA NA NA NA #> .. ..$ .weight_censored: num [1:5] 1.01 NA NA NA NA #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.918 0.803 0.682 0.568 0.464 #> .. ..$ .weight_time : num [1:5] 100 167 167 167 167 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.863 0.684 0.516 0.375 0.264 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.824 0.606 0.418 0.275 0.173 #> .. ..$ .weight_time : num [1:5] 100 200 284 284 284 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.793 0.793 0.793 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.26 1.26 1.26 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.903 0.768 0.632 0.506 0.397 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.896 0.753 0.61 0.48 0.37 #> .. ..$ .weight_time : num [1:5] 100 147 147 147 147 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.918 0.803 0.682 0.568 0.464 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7321 0.4471 0.2462 0.1253 0.0597 #> .. ..$ .weight_time : num [1:5] 100 163 163 163 163 #> .. ..$ .pred_censored : num [1:5] 0.995 0.99 0.99 0.99 0.99 #> .. ..$ .weight_censored: num [1:5] 1.01 1.01 1.01 1.01 1.01 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.894 0.748 0.604 0.473 0.362 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.825 0.609 0.422 0.278 0.176 #> .. ..$ .weight_time : num [1:5] 100 200 239 239 239 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.86 0.86 0.86 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.16 1.16 1.16 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.832 0.621 0.437 0.293 0.189 #> .. ..$ .weight_time : num [1:5] 88 88 88 88 88 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.907 0.778 0.646 0.524 0.416 #> .. ..$ .weight_time : num [1:5] 100 200 245 245 245 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.845 0.845 0.845 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.18 1.18 1.18 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.933 0.837 0.733 0.632 0.536 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.7321 0.4471 0.2462 0.1253 0.0597 #> .. ..$ .weight_time : num [1:5] 30 30 30 30 30 #> .. ..$ .pred_censored : num [1:5] 1 1 1 1 1 #> .. ..$ .weight_censored: num [1:5] 1 1 1 1 1 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.827 0.612 0.426 0.282 0.179 #> .. ..$ .weight_time : num [1:5] 100 179 179 179 179 #> .. ..$ .pred_censored : num [1:5] 0.995 0.966 0.966 0.966 0.966 #> .. ..$ .weight_censored: num [1:5] 1.01 1.03 1.03 1.03 1.03 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.824 0.606 0.418 0.275 0.173 #> .. ..$ .weight_time : num [1:5] 100 200 300 310 310 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.735 0.735 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.36 1.36 #> ..$ : tibble [5 × 5] (S3: tbl_df/tbl/data.frame) #> .. ..$ .eval_time : num [1:5] 100 200 300 400 500 #> .. ..$ .pred_survival : num [1:5] 0.835 0.627 0.444 0.3 0.196 #> .. ..$ .weight_time : num [1:5] 100 200 300 400 477 #> .. ..$ .pred_censored : num [1:5] 0.995 0.928 0.76 0.663 0.613 #> .. ..$ .weight_censored: num [1:5] 1.01 1.08 1.31 1.51 1.63 #> .. [list output truncated] #> $ .pred_time: num [1:228] 324 476 521 368 506 ... #> $ surv_obj : 'Surv' num [1:228, 1:2] 306 455 1010+ 210 883 1022+ 310 361 218 166 ... #> ..- attr(*, \"dimnames\")=List of 2 #> .. ..$ : NULL #> .. ..$ : chr [1:2] \"time\" \"status\" #> ..- attr(*, \"type\")= chr \"right\" # `surv_obj` is a `Surv()` object"},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute error — mae","title":"Mean absolute error — mae","text":"Calculate mean absolute error. metric units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute error — mae","text":"","code":"mae(data, ...) # S3 method for data.frame mae(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mae_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute error — mae","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute error — mae","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mae_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute error — mae","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mae.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute error — mae","text":"","code":"# Supply truth and predictions as bare column names mae(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mae standard 0.545 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mae(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mae standard 0.512 #> 2 10 mae standard 0.504 #> 3 2 mae standard 0.545 #> 4 3 mae standard 0.496 #> 5 4 mae standard 0.587 #> 6 5 mae standard 0.500 #> 7 6 mae standard 0.627 #> 8 7 mae standard 0.566 #> 9 8 mae standard 0.473 #> 10 9 mae standard 0.540 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.535"},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute percent error — mape","title":"Mean absolute percent error — mape","text":"Calculate mean absolute percentage error. metric relative units.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute percent error — mape","text":"","code":"mape(data, ...) # S3 method for data.frame mape(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mape_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute percent error — mape","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute percent error — mape","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mape_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean absolute percent error — mape","text":"Note value Inf returned mape() observed value negative.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute percent error — mape","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mape.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute percent error — mape","text":"","code":"# Supply truth and predictions as bare column names mape(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mape standard Inf library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mape(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mape standard Inf #> 2 10 mape standard Inf #> 3 2 mape standard Inf #> 4 3 mape standard 28.5 #> 5 4 mape standard 95.3 #> 6 5 mape standard Inf #> 7 6 mape standard 73.1 #> 8 7 mape standard Inf #> 9 8 mape standard Inf #> 10 9 mape standard 37.9 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 Inf"},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean absolute scaled error — mase","title":"Mean absolute scaled error — mase","text":"Calculate mean absolute scaled error. metric scale independent symmetric. generally used comparing forecast error time series settings. Due time series nature metric, necessary order observations ascending order time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean absolute scaled error — mase","text":"","code":"mase(data, ...) # S3 method for data.frame mase( data, truth, estimate, m = 1L, mae_train = NULL, na_rm = TRUE, case_weights = NULL, ... ) mase_vec( truth, estimate, m = 1L, mae_train = NULL, na_rm = TRUE, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean absolute scaled error — mase","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. m integer value number lags used calculate -sample seasonal naive error. default used non-seasonal time series. observation daily level data showed weekly seasonality, m = 7L reasonable choice 7-day seasonal naive calculation. mae_train numeric value allows user provide -sample seasonal naive mean absolute error. value provided, --sample seasonal naive mean absolute error calculated truth used instead. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean absolute scaled error — mase","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mase_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean absolute scaled error — mase","text":"mase() different numeric metrics. original implementation mase() calls using -sample naive mean absolute error compute scaled errors . uses instead --sample error chance --sample error computed forecasting short horizon (.e. sample size 1 2). However, yardstick knows --sample truth estimate values. , --sample error used computation default. -sample naive mean absolute error required known, can passed mae_train argument used instead. -sample data available, naive mean absolute error can easily computed mae(data, truth, lagged_truth).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Mean absolute scaled error — mase","text":"Rob J. Hyndman (2006). ANOTHER LOOK FORECAST-ACCURACY METRICS INTERMITTENT DEMAND. Foresight, 4, 46.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean absolute scaled error — mase","text":"Alex Hallam","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mase.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean absolute scaled error — mase","text":"","code":"# Supply truth and predictions as bare column names mase(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mase(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mase standard 0.256 #> 2 10 mase standard 0.240 #> 3 2 mase standard 0.238 #> 4 3 mase standard 0.219 #> 5 4 mase standard 0.229 #> 6 5 mase standard 0.261 #> 7 6 mase standard 0.217 #> 8 7 mase standard 0.267 #> 9 8 mase standard 0.216 #> 10 9 mase standard 0.251 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.240"},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":null,"dir":"Reference","previous_headings":"","what":"Matthews correlation coefficient — mcc","title":"Matthews correlation coefficient — mcc","text":"Matthews correlation coefficient","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Matthews correlation coefficient — mcc","text":"","code":"mcc(data, ...) # S3 method for data.frame mcc(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mcc_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Matthews correlation coefficient — mcc","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Matthews correlation coefficient — mcc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mcc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Matthews correlation coefficient — mcc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Matthews correlation coefficient — mcc","text":"mcc() known multiclass generalization computed automatically factor 2 levels provided. , averaging methods provided.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Matthews correlation coefficient — mcc","text":"Giuseppe, J. (2012). \"Comparison MCC CEN Error Measures Multi-Class Prediction\". PLOS ONE. Vol 7, Iss 8, e41882.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Matthews correlation coefficient — mcc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mcc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Matthews correlation coefficient — mcc","text":"","code":"library(dplyr) data(\"two_class_example\") data(\"hpc_cv\") # Two class mcc(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mcc binary 0.677 # Multiclass # mcc() has a natural multiclass extension hpc_cv %>% filter(Resample == \"Fold01\") %>% mcc(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mcc multiclass 0.542 # Groups are respected hpc_cv %>% group_by(Resample) %>% mcc(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 mcc multiclass 0.542 #> 2 Fold02 mcc multiclass 0.521 #> 3 Fold03 mcc multiclass 0.602 #> 4 Fold04 mcc multiclass 0.519 #> 5 Fold05 mcc multiclass 0.520 #> 6 Fold06 mcc multiclass 0.494 #> 7 Fold07 mcc multiclass 0.461 #> 8 Fold08 mcc multiclass 0.538 #> 9 Fold09 mcc multiclass 0.459 #> 10 Fold10 mcc multiclass 0.498"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for summarizing new metrics — metric-summarizers","title":"Developer function for summarizing new metrics — metric-summarizers","text":"numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), static_survival_metric_summarizer() useful alongside check_metric yardstick_remove_missing implementing new custom metrics. functions call metric function inside dplyr::summarise() dplyr::reframe() curve_metric_summarizer(). See Custom performance metrics information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for summarizing new metrics — metric-summarizers","text":"","code":"numeric_metric_summarizer( name, fn, data, truth, estimate, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) class_metric_summarizer( name, fn, data, truth, estimate, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) prob_metric_summarizer( name, fn, data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) curve_metric_summarizer( name, fn, data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, fn_options = list(), error_call = caller_env() ) dynamic_survival_metric_summarizer( name, fn, data, truth, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) static_survival_metric_summarizer( name, fn, data, truth, estimate, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() ) curve_survival_metric_summarizer( name, fn, data, truth, ..., na_rm = TRUE, case_weights = NULL, fn_options = list(), error_call = caller_env() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for summarizing new metrics — metric-summarizers","text":"name single character representing name metric use tibble output. modified include type averaging appropriate. fn vector version custom metric function. generally takes truth, estimate, na_rm, extra arguments needed calculate metric. data data frame truth estimate columns passed data frame version metric function called numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), static_survival_metric_summarizer(). truth unquoted column name corresponding truth column. estimate Generally, unquoted column name corresponding estimate column. metrics take multiple columns ... like class probability metrics, result dots_to_estimate(). ... dots future extensions must empty. na_rm logical value indicating whether NA values stripped computation proceeds. removal executed yardstick_remove_missing(). case_weights metrics supporting case weights, unquoted column name corresponding case weights can passed . NULL, case weights passed fn named argument case_weights. fn_options named list metric specific options. spliced metric function call using !!! rlang. default results nothing spliced call. error_call execution environment currently running function, e.g. caller_env(). function mentioned error messages source error. See call argument abort() information. estimator can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use. event_level can either NULL use default event_level value fn single string either \"first\" \"second\" pass along describing level considered \"event\".","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric-summarizers.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Developer function for summarizing new metrics — metric-summarizers","text":"numeric_metric_summarizer(), class_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer(), dynamic_survival_metric_summarizer(), dynamic_survival_metric_summarizer() generally called data frame version metric function. knows call metric grouped data frames returns tibble consistent metrics.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":null,"dir":"Reference","previous_headings":"","what":"Combine metric functions — metric_set","title":"Combine metric functions — metric_set","text":"metric_set() allows combine multiple metric functions together new function calculates .","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Combine metric functions — metric_set","text":"","code":"metric_set(...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Combine metric functions — metric_set","text":"... bare names functions included metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Combine metric functions — metric_set","text":"functions must either: numeric metrics mix class metrics class prob metrics mix dynamic, integrated, static survival metrics instance, rmse() can used mae() numeric metrics, accuracy() classification metric. accuracy() can used roc_auc(). returned metric function different argument list depending whether numeric metrics mix class/prob metrics passed . mixing class class prob metrics, pass hard predictions (factor column) named argument estimate, soft predictions (class probability columns) bare column names tidyselect selectors .... mixing dynamic, integrated, static survival metrics, pass time predictions named argument estimate, survival predictions bare column names tidyselect selectors .... metric_tweak() used \"tweak\" one arguments, like estimator event_level, tweaked version wins. allows set estimator metric metric basis still use metric_set().","code":"# Numeric metric set signature: fn( data, truth, estimate, na_rm = TRUE, case_weights = NULL, ... ) # Class / prob metric set signature: fn( data, truth, ..., estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) # Dynamic / integrated / static survival metric set signature: fn( data, truth, ..., estimate, na_rm = TRUE, case_weights = NULL )"},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_set.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Combine metric functions — metric_set","text":"","code":"library(dplyr) # Multiple regression metrics multi_metric <- metric_set(rmse, rsq, ccc) # The returned function has arguments: # fn(data, truth, estimate, na_rm = TRUE, ...) multi_metric(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 ccc standard 0.937 # Groups are respected on the new metric function class_metrics <- metric_set(accuracy, kap) hpc_cv %>% group_by(Resample) %>% class_metrics(obs, estimate = pred) #> # A tibble: 20 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492 # --------------------------------------------------------------------------- # If you need to set options for certain metrics, # do so by wrapping the metric and setting the options inside the wrapper, # passing along truth and estimate as quoted arguments. # Then add on the function class of the underlying wrapped function, # and the direction of optimization. ccc_with_bias <- function(data, truth, estimate, na_rm = TRUE, ...) { ccc( data = data, truth = !!rlang::enquo(truth), estimate = !!rlang::enquo(estimate), # set bias = TRUE bias = TRUE, na_rm = na_rm, ... ) } # Use `new_numeric_metric()` to formalize this new metric function ccc_with_bias <- new_numeric_metric(ccc_with_bias, \"maximize\") multi_metric2 <- metric_set(rmse, rsq, ccc_with_bias) multi_metric2(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 ccc standard 0.937 # --------------------------------------------------------------------------- # A class probability example: # Note that, when given class or class prob functions, # metric_set() returns a function with signature: # fn(data, truth, ..., estimate) # to be able to mix class and class prob metrics. # You must provide the `estimate` column by explicitly naming # the argument class_and_probs_metrics <- metric_set(roc_auc, pr_auc, accuracy) hpc_cv %>% group_by(Resample) %>% class_and_probs_metrics(obs, VF:L, estimate = pred) #> # A tibble: 30 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> # ℹ 20 more rows"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for summarizing new metrics — metric_summarizer","title":"Developer function for summarizing new metrics — metric_summarizer","text":"metric_summarizer() soft-deprecated yardstick 1.2.0. Please switch use class_metric_summarizer(), numeric_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for summarizing new metrics — metric_summarizer","text":"","code":"metric_summarizer( metric_nm, metric_fn, data, truth, estimate, estimator = NULL, na_rm = TRUE, event_level = NULL, case_weights = NULL, ..., metric_fn_options = list() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_summarizer.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for summarizing new metrics — metric_summarizer","text":"metric_nm single character representing name metric use tibble output. modified include type averaging appropriate. metric_fn vector version custom metric function. generally takes truth, estimate, na_rm, extra arguments needed calculate metric. data data frame truth estimate columns passed data frame version metric function called metric_summarizer(). truth unquoted column name corresponding truth column. estimate Generally, unquoted column name corresponding estimate column. metrics take multiple columns ... like class probability metrics, result dots_to_estimate(). estimator numeric metrics, left NULL averaging passed metric function implementation. classification metrics, can either NULL default auto-selection averaging (\"binary\" \"macro\"), single character pass along metric implementation describing kind averaging use. na_rm logical value indicating whether NA values stripped computation proceeds. removal executed metric_vec_template(). event_level numeric metrics, left NULL prevent passed metric function implementation. classification metrics, can either NULL use default event_level value metric_fn single string either \"first\" \"second\" pass along describing level considered \"event\". case_weights metrics supporting case weights, unquoted column name corresponding case weights can passed . NULL, case weights passed metric_fn named argument case_weights. ... Currently used. Metric specific options passed metric_fn_options. metric_fn_options named list metric specific options. spliced metric function call using !!! rlang. default results nothing spliced call.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":null,"dir":"Reference","previous_headings":"","what":"Tweak a metric function — metric_tweak","title":"Tweak a metric function — metric_tweak","text":"metric_tweak() allows tweak existing metric .fn, giving new .name setting new optional argument defaults .... similar purrr::partial(), designed specifically yardstick metrics. metric_tweak() especially useful constructing metric_set() tuning tune package. metric set constructed, way adjust value optional arguments (beta f_meas()). Using metric_tweak(), can set optional arguments custom values ahead time, go metric set.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Tweak a metric function — metric_tweak","text":"","code":"metric_tweak(.name, .fn, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Tweak a metric function — metric_tweak","text":".name single string giving name new metric. used \".metric\" column output. .fn existing yardstick metric function tweak. ... Name-value pairs specifying optional arguments override values replace . Arguments data, truth, estimate considered protected, overridden, optional arguments can altered.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Tweak a metric function — metric_tweak","text":"tweaked version .fn, updated use new defaults supplied ....","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Tweak a metric function — metric_tweak","text":"function returned metric_tweak() takes ... arguments, passed original .fn. Passing data, truth, estimate position generally safe, recommended pass optional arguments name ensure evaluated correctly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_tweak.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Tweak a metric function — metric_tweak","text":"","code":"mase12 <- metric_tweak(\"mase12\", mase, m = 12) # Defaults to `m = 1` mase(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 # Updated to use `m = 12`. `mase12()` has this set already. mase(solubility_test, solubility, prediction, m = 12) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 0.582 mase12(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mase12 standard 0.582 # This is most useful to set optional argument values ahead of time when # using a metric set mase10 <- metric_tweak(\"mase10\", mase, m = 10) metrics <- metric_set(mase, mase10, mase12) metrics(solubility_test, solubility, prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 mase standard 3.56 #> 2 mase10 standard 0.664 #> 3 mase12 standard 0.582"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for calling new metrics — metric_vec_template","title":"Developer function for calling new metrics — metric_vec_template","text":"metric_vec_template() soft-deprecated yardstick 1.2.0. Please switch use check_metric yardstick_remove_missing functions.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for calling new metrics — metric_vec_template","text":"","code":"metric_vec_template( metric_impl, truth, estimate, na_rm = TRUE, cls = \"numeric\", estimator = NULL, case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/metric_vec_template.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for calling new metrics — metric_vec_template","text":"metric_impl core implementation function custom metric. core implementation function generally defined inside vector method metric function. truth realized vector truth. either factor numeric. estimate realized estimate result. either numeric vector, factor vector, numeric matrix (case multiple class probability columns) depending metric function. na_rm logical value indicating whether NA values stripped computation proceeds. NA values removed getting core implementation function worry handling . na_rm=FALSE NA values exist, NA automatically returned. cls character vector length 1 2 corresponding class truth estimate , respectively. truth estimate class, just supply vector length 1. different, supply vector length 2. matrices, best supply \"numeric\" class check . estimator type averaging use. point, averaging type finalized, character vector length 1\\. default, character value required one : \"binary\", \"macro\", \"micro\", \"macro_weighted\". metric allows less averaging methods, override averaging_override. case_weights Optionally, realized case weights, numeric vector. must length truth, considered na_rm checks. supplied, passed metric_impl named argument case_weights. ... Extra arguments core metric function, metric_impl, can technically passed , generally extra args added R's scoping rules core metric function created fly vector method called.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":null,"dir":"Reference","previous_headings":"","what":"General Function to Estimate Performance — metrics","title":"General Function to Estimate Performance — metrics","text":"function estimates one common performance estimates depending class truth (see Value ) returns three column tibble.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"General Function to Estimate Performance — metrics","text":"","code":"metrics(data, ...) # S3 method for data.frame metrics(data, truth, estimate, ..., na_rm = TRUE, options = list())"},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"General Function to Estimate Performance — metrics","text":"data data.frame containing columns specified truth, estimate, .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true results (numeric factor). unquoted column name although argument passed expression support quasiquotation (can unquote column names). estimate column identifier predicted results (also numeric factor). truth can specified different ways primary method use unquoted variable name. na_rm logical value indicating whether NA values stripped computation proceeds. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"General Function to Estimate Performance — metrics","text":"three column tibble. truth factor, rows accuracy() Kappa statistic (kap()). truth two levels 1 column class probabilities passed ..., rows two class versions mn_log_loss() roc_auc(). truth two levels full set class probabilities passed ..., rows multiclass version mn_log_loss() Hand Till generalization roc_auc(). truth numeric, rows rmse(), rsq(), mae().","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/metrics.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"General Function to Estimate Performance — metrics","text":"","code":"# Accuracy and kappa metrics(two_class_example, truth, predicted) #> # A tibble: 2 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 # Add on multinomal log loss and ROC AUC by specifying class prob columns metrics(two_class_example, truth, predicted, Class1) #> # A tibble: 4 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 mn_log_loss binary 0.328 #> 4 roc_auc binary 0.939 # Regression metrics metrics(solubility_test, truth = solubility, estimate = prediction) #> # A tibble: 3 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 #> 2 rsq standard 0.879 #> 3 mae standard 0.545 # Multiclass metrics work, but you cannot specify any averaging # for roc_auc() besides the default, hand_till. Use the specific function # if you need more customization library(dplyr) hpc_cv %>% group_by(Resample) %>% metrics(obs, pred, VF:L) %>% print(n = 40) #> # A tibble: 40 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 accuracy multiclass 0.726 #> 2 Fold02 accuracy multiclass 0.712 #> 3 Fold03 accuracy multiclass 0.758 #> 4 Fold04 accuracy multiclass 0.712 #> 5 Fold05 accuracy multiclass 0.712 #> 6 Fold06 accuracy multiclass 0.697 #> 7 Fold07 accuracy multiclass 0.675 #> 8 Fold08 accuracy multiclass 0.721 #> 9 Fold09 accuracy multiclass 0.673 #> 10 Fold10 accuracy multiclass 0.699 #> 11 Fold01 kap multiclass 0.533 #> 12 Fold02 kap multiclass 0.512 #> 13 Fold03 kap multiclass 0.594 #> 14 Fold04 kap multiclass 0.511 #> 15 Fold05 kap multiclass 0.514 #> 16 Fold06 kap multiclass 0.486 #> 17 Fold07 kap multiclass 0.454 #> 18 Fold08 kap multiclass 0.531 #> 19 Fold09 kap multiclass 0.454 #> 20 Fold10 kap multiclass 0.492 #> 21 Fold01 mn_log_loss multiclass 0.734 #> 22 Fold02 mn_log_loss multiclass 0.808 #> 23 Fold03 mn_log_loss multiclass 0.705 #> 24 Fold04 mn_log_loss multiclass 0.747 #> 25 Fold05 mn_log_loss multiclass 0.799 #> 26 Fold06 mn_log_loss multiclass 0.766 #> 27 Fold07 mn_log_loss multiclass 0.927 #> 28 Fold08 mn_log_loss multiclass 0.855 #> 29 Fold09 mn_log_loss multiclass 0.861 #> 30 Fold10 mn_log_loss multiclass 0.821 #> 31 Fold01 roc_auc hand_till 0.813 #> 32 Fold02 roc_auc hand_till 0.817 #> 33 Fold03 roc_auc hand_till 0.869 #> 34 Fold04 roc_auc hand_till 0.849 #> 35 Fold05 roc_auc hand_till 0.811 #> 36 Fold06 roc_auc hand_till 0.836 #> 37 Fold07 roc_auc hand_till 0.825 #> 38 Fold08 roc_auc hand_till 0.846 #> 39 Fold09 roc_auc hand_till 0.828 #> 40 Fold10 roc_auc hand_till 0.812"},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean log loss for multinomial data — mn_log_loss","title":"Mean log loss for multinomial data — mn_log_loss","text":"Compute logarithmic loss classification model.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean log loss for multinomial data — mn_log_loss","text":"","code":"mn_log_loss(data, ...) # S3 method for data.frame mn_log_loss( data, truth, ..., na_rm = TRUE, sum = FALSE, event_level = yardstick_event_level(), case_weights = NULL ) mn_log_loss_vec( truth, estimate, na_rm = TRUE, sum = FALSE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean log loss for multinomial data — mn_log_loss","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. sum logical. sum likelihood contributions returned (instead mean value)? event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean log loss for multinomial data — mn_log_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mn_log_loss_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean log loss for multinomial data — mn_log_loss","text":"Log loss measure performance classification model. perfect model log loss 0. Compared accuracy(), log loss takes account uncertainty prediction gives detailed view actual performance. example, given two input probabilities .6 .9 classified predicting positive value, say, \"Yes\", accuracy metric interpret value. true output \"Yes\", log loss penalizes .6 \"less sure\" result compared probability .9.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Mean log loss for multinomial data — mn_log_loss","text":"Log loss known multiclass extension, simply sum log loss values class prediction. , averaging types supported.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean log loss for multinomial data — mn_log_loss","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mn_log_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean log loss for multinomial data — mn_log_loss","text":"","code":"# Two class data(\"two_class_example\") mn_log_loss(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328 # Multiclass library(dplyr) data(hpc_cv) # You can use the col1:colN tidyselect syntax hpc_cv %>% filter(Resample == \"Fold01\") %>% mn_log_loss(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss multiclass 0.734 # Groups are respected hpc_cv %>% group_by(Resample) %>% mn_log_loss(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 mn_log_loss multiclass 0.734 #> 2 Fold02 mn_log_loss multiclass 0.808 #> 3 Fold03 mn_log_loss multiclass 0.705 #> 4 Fold04 mn_log_loss multiclass 0.747 #> 5 Fold05 mn_log_loss multiclass 0.799 #> 6 Fold06 mn_log_loss multiclass 0.766 #> 7 Fold07 mn_log_loss multiclass 0.927 #> 8 Fold08 mn_log_loss multiclass 0.855 #> 9 Fold09 mn_log_loss multiclass 0.861 #> 10 Fold10 mn_log_loss multiclass 0.821 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") mn_log_loss_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.7338423 # Supply `...` with quasiquotation prob_cols <- levels(two_class_example$truth) mn_log_loss(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328 mn_log_loss(two_class_example, truth, !!prob_cols[1]) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mn_log_loss binary 0.328"},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean percentage error — mpe","title":"Mean percentage error — mpe","text":"Calculate mean percentage error. metric relative units. can used measure estimate's bias. Note truth values 0, value : -Inf (estimate > 0), Inf (estimate < 0), NaN (estimate == 0) returned mpe().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean percentage error — mpe","text":"","code":"mpe(data, ...) # S3 method for data.frame mpe(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) mpe_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean percentage error — mpe","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean percentage error — mpe","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. mpe_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean percentage error — mpe","text":"Thomas Bierhance","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/mpe.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean percentage error — mpe","text":"","code":"# `solubility_test$solubility` has zero values with corresponding # `$prediction` values that are negative. By definition, this causes `Inf` # to be returned from `mpe()`. solubility_test[solubility_test$solubility == 0, ] #> solubility prediction #> 17 0 -0.1532030 #> 220 0 -0.3876578 mpe(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mpe standard Inf # We'll remove the zero values for demonstration solubility_test <- solubility_test[solubility_test$solubility != 0, ] # Supply truth and predictions as bare column names mpe(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 mpe standard 16.1 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% mpe(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 mpe standard -56.2 #> 2 10 mpe standard 50.4 #> 3 2 mpe standard -27.9 #> 4 3 mpe standard 0.470 #> 5 4 mpe standard -0.836 #> 6 5 mpe standard -35.3 #> 7 6 mpe standard 7.51 #> 8 7 mpe standard -34.5 #> 9 8 mpe standard 7.87 #> 10 9 mpe standard 14.7 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 -7.38"},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean signed deviation — msd","title":"Mean signed deviation — msd","text":"Mean signed deviation (also known mean signed difference, mean signed error) computes average differences truth estimate. related metric mean absolute error (mae()).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean signed deviation — msd","text":"","code":"msd(data, ...) # S3 method for data.frame msd(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) msd_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean signed deviation — msd","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean signed deviation — msd","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. msd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Mean signed deviation — msd","text":"Mean signed deviation rarely used, since positive negative errors cancel . example, msd_vec(c(100, -100), c(0, 0)) return seemingly \"perfect\" value 0, even though estimate wildly different truth. mae() attempts remedy taking absolute value differences computing mean. metric computed mean(truth - estimate), following convention \"error\" computed observed - predicted. expected metric computed mean(estimate - truth), reverse sign result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean signed deviation — msd","text":"Thomas Bierhance","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/msd.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean signed deviation — msd","text":"","code":"# Supply truth and predictions as bare column names msd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 msd standard -0.0143 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% msd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 msd standard -0.0119 #> 2 10 msd standard -0.0424 #> 3 2 msd standard 0.0111 #> 4 3 msd standard -0.0906 #> 5 4 msd standard -0.0859 #> 6 5 msd standard -0.0301 #> 7 6 msd standard -0.0132 #> 8 7 msd standard -0.00640 #> 9 8 msd standard -0.000697 #> 10 9 msd standard -0.0399 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 -0.0310"},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Construct a new metric function — new-metric","title":"Construct a new metric function — new-metric","text":"functions provide convenient wrappers create three types metric functions yardstick: numeric metrics, class metrics, class probability metrics. add metric-specific class fn attach direction attribute. features used metric_set() tune model tuning. See Custom performance metrics information creating custom metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Construct a new metric function — new-metric","text":"","code":"new_class_metric(fn, direction) new_prob_metric(fn, direction) new_numeric_metric(fn, direction) new_dynamic_survival_metric(fn, direction) new_integrated_survival_metric(fn, direction) new_static_survival_metric(fn, direction)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new-metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Construct a new metric function — new-metric","text":"fn function. metric function attach metric-specific class direction attribute . direction string. One : \"maximize\" \"minimize\" \"zero\"","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":null,"dir":"Reference","previous_headings":"","what":"Create groupwise metrics — new_groupwise_metric","title":"Create groupwise metrics — new_groupwise_metric","text":"Groupwise metrics quantify disparity value metric across number groups. Groupwise metrics value zero indicate underlying metric equal across groups. yardstick defines several common fairness metrics using function, demographic_parity(), equal_opportunity(), equalized_odds().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Create groupwise metrics — new_groupwise_metric","text":"","code":"new_groupwise_metric(fn, name, aggregate, direction = \"minimize\")"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Create groupwise metrics — new_groupwise_metric","text":"fn yardstick metric function metric set. name name metric place .metric column output. aggregate function summarize generated metric set results. function takes metric set results first argument returns single numeric giving .estimate value output. See Value Examples sections example uses. direction string. One : \"maximize\" \"minimize\" \"zero\"","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Create groupwise metrics — new_groupwise_metric","text":"function function factory; output function. , functions function outputs also function factories. explicitly, looks like: outputted dem_parity function takes one argument, , indicating data-masked variable giving sensitive feature. called argument, dem_parity return yardstick metric function like : Note dem_parity take arguments , thus knows nothing data applied column name \"gender\" . output dem_parity_by_gender metric function takes arguments function supplied fn, case detection_prevalence. thus interface like yardstick function except look \"gender\" column data supplied. addition examples , see documentation return value fairness metrics like demographic_parity(), equal_opportunity(), equalized_odds() learn output function can used.","code":"# a function with similar implementation to `demographic_parity()`: diff_range <- function(x) {diff(range(x$.estimate))} dem_parity <- new_groupwise_metric( fn = detection_prevalence, name = \"dem_parity\", aggregate = diff_range ) dem_parity_by_gender <- dem_parity(gender)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Create groupwise metrics — new_groupwise_metric","text":"Note yardstick metrics group-aware , passed grouped data, return metric values calculated group. passed grouped data, groupwise metrics also return metric values group, metric values calculated first additionally grouping variable passed summarizing per-group metric estimates across groups using function passed aggregate argument. Learn grouping behavior yardstick using vignette(\"grouping\", \"yardstick\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"relevant-group-level","dir":"Reference","previous_headings":"","what":"Relevant Group Level","title":"Create groupwise metrics — new_groupwise_metric","text":"Additional arguments can passed function outputted function function outputs. : finer control groups treated, use aggregate argument.","code":"res_fairness <- new_groupwise_metric(...) res_by <- res_fairness(by) res_by(..., additional_arguments_to_aggregate = TRUE)"},{"path":"https://yardstick.tidymodels.org/dev/reference/new_groupwise_metric.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Create groupwise metrics — new_groupwise_metric","text":"","code":"data(hpc_cv) # `demographic_parity`, among other fairness metrics, # is generated with `new_groupwise_metric()`: diff_range <- function(x) {diff(range(x$.estimate))} demographic_parity_ <- new_groupwise_metric( fn = detection_prevalence, name = \"demographic_parity\", aggregate = diff_range ) m_set <- metric_set(demographic_parity_(Resample)) m_set(hpc_cv, truth = obs, estimate = pred) #> # A tibble: 1 × 4 #> .metric .by .estimator .estimate #> #> 1 demographic_parity Resample macro 2.78e-17 # the `post` argument can be used to accommodate a wide # variety of parameterizations. to encode demographic # parity as a ratio inside of a difference, for example: ratio_range <- function(x, ...) { range <- range(x$.estimate) range[1] / range[2] } demographic_parity_ratio <- new_groupwise_metric( fn = detection_prevalence, name = \"demographic_parity_ratio\", aggregate = ratio_range )"},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":null,"dir":"Reference","previous_headings":"","what":"Negative predictive value — npv","title":"Negative predictive value — npv","text":"functions calculate npv() (negative predictive value) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), sens(), ppv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Negative predictive value — npv","text":"","code":"npv(data, ...) # S3 method for data.frame npv( data, truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) npv_vec( truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Negative predictive value — npv","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. prevalence numeric value rate \"positive\" class data. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Negative predictive value — npv","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. npv_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Negative predictive value — npv","text":"positive predictive value (ppv()) defined percent predicted positives actually positive negative predictive value (npv()) defined percent negative positives actually negative.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Negative predictive value — npv","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Negative predictive value — npv","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Negative predictive value — npv","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Negative predictive value — npv","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 2: predictive values,'' British Medical Journal, vol 309, 102.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Negative predictive value — npv","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/npv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Negative predictive value — npv","text":"","code":"# Two class data(\"two_class_example\") npv(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 npv binary 0.861 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% npv(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 npv macro 0.906 # Groups are respected hpc_cv %>% group_by(Resample) %>% npv(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 npv macro 0.906 #> 2 Fold02 npv macro 0.901 #> 3 Fold03 npv macro 0.917 #> 4 Fold04 npv macro 0.897 #> 5 Fold05 npv macro 0.897 #> 6 Fold06 npv macro 0.892 #> 7 Fold07 npv macro 0.882 #> 8 Fold08 npv macro 0.902 #> 9 Fold09 npv macro 0.879 #> 10 Fold10 npv macro 0.890 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% npv(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 npv macro_weighted 0.896 #> 2 Fold02 npv macro_weighted 0.890 #> 3 Fold03 npv macro_weighted 0.905 #> 4 Fold04 npv macro_weighted 0.878 #> 5 Fold05 npv macro_weighted 0.878 #> 6 Fold06 npv macro_weighted 0.871 #> 7 Fold07 npv macro_weighted 0.853 #> 8 Fold08 npv macro_weighted 0.885 #> 9 Fold09 npv macro_weighted 0.845 #> 10 Fold10 npv macro_weighted 0.864 # Vector version npv_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8609865 # Making Class2 the \"relevant\" level npv_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8194946"},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":null,"dir":"Reference","previous_headings":"","what":"Liver Pathology Data — pathology","title":"Liver Pathology Data — pathology","text":"Liver Pathology Data","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Liver Pathology Data — pathology","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Liver Pathology Data — pathology","text":"pathology data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Liver Pathology Data — pathology","text":"data results x-ray examination determine whether liver abnormal (scan column) versus extensive pathology results approximate truth (pathology).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pathology.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Liver Pathology Data — pathology","text":"","code":"data(pathology) str(pathology) #> 'data.frame':\t344 obs. of 2 variables: #> $ pathology: Factor w/ 2 levels \"abnorm\",\"norm\": 1 1 1 1 1 1 1 1 1 1 ... #> $ scan : Factor w/ 2 levels \"abnorm\",\"norm\": 1 1 1 1 1 1 1 1 1 1 ..."},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":null,"dir":"Reference","previous_headings":"","what":"Mean log loss for Poisson data — poisson_log_loss","title":"Mean log loss for Poisson data — poisson_log_loss","text":"Calculate loss function Poisson distribution.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Mean log loss for Poisson data — poisson_log_loss","text":"","code":"poisson_log_loss(data, ...) # S3 method for data.frame poisson_log_loss(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) poisson_log_loss_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Mean log loss for Poisson data — poisson_log_loss","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true counts (integer). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, integer vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Mean log loss for Poisson data — poisson_log_loss","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. poisson_log_loss_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Mean log loss for Poisson data — poisson_log_loss","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/poisson_log_loss.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Mean log loss for Poisson data — poisson_log_loss","text":"","code":"count_truth <- c(2L, 7L, 1L, 1L, 0L, 3L) count_pred <- c(2.14, 5.35, 1.65, 1.56, 1.3, 2.71) count_results <- dplyr::tibble(count = count_truth, pred = count_pred) # Supply truth and predictions as bare column names poisson_log_loss(count_results, count, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 poisson_log_loss standard 1.42"},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":null,"dir":"Reference","previous_headings":"","what":"Positive predictive value — ppv","title":"Positive predictive value — ppv","text":"functions calculate ppv() (positive predictive value) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), sens(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Positive predictive value — ppv","text":"","code":"ppv(data, ...) # S3 method for data.frame ppv( data, truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) ppv_vec( truth, estimate, prevalence = NULL, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Positive predictive value — ppv","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. prevalence numeric value rate \"positive\" class data. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Positive predictive value — ppv","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. ppv_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Positive predictive value — ppv","text":"positive predictive value (ppv()) defined percent predicted positives actually positive negative predictive value (npv()) defined percent negative positives actually negative.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Positive predictive value — ppv","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Positive predictive value — ppv","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Positive predictive value — ppv","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Positive predictive value — ppv","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 2: predictive values,'' British Medical Journal, vol 309, 102.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Positive predictive value — ppv","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/ppv.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Positive predictive value — ppv","text":"","code":"# Two class data(\"two_class_example\") ppv(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv binary 0.819 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% ppv(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv macro 0.637 # Groups are respected hpc_cv %>% group_by(Resample) %>% ppv(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 ppv macro 0.637 #> 2 Fold02 ppv macro 0.603 #> 3 Fold03 ppv macro 0.706 #> 4 Fold04 ppv macro 0.658 #> 5 Fold05 ppv macro 0.651 #> 6 Fold06 ppv macro 0.626 #> 7 Fold07 ppv macro 0.562 #> 8 Fold08 ppv macro 0.652 #> 9 Fold09 ppv macro 0.605 #> 10 Fold10 ppv macro 0.625 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% ppv(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 ppv macro_weighted 0.697 #> 2 Fold02 ppv macro_weighted 0.690 #> 3 Fold03 ppv macro_weighted 0.752 #> 4 Fold04 ppv macro_weighted 0.690 #> 5 Fold05 ppv macro_weighted 0.705 #> 6 Fold06 ppv macro_weighted 0.682 #> 7 Fold07 ppv macro_weighted 0.649 #> 8 Fold08 ppv macro_weighted 0.702 #> 9 Fold09 ppv macro_weighted 0.661 #> 10 Fold10 ppv macro_weighted 0.683 # Vector version ppv_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8194946 # Making Class2 the \"relevant\" level ppv_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8609865 # But what if we think that Class 1 only occurs 40% of the time? ppv(two_class_example, truth, predicted, prevalence = 0.40) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 ppv binary 0.740"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the precision recall curve — pr_auc","title":"Area under the precision recall curve — pr_auc","text":"pr_auc() metric computes area precision recall curve. See pr_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the precision recall curve — pr_auc","text":"","code":"pr_auc(data, ...) # S3 method for data.frame pr_auc( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL ) pr_auc_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the precision recall curve — pr_auc","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. two general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based truth. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the precision recall curve — pr_auc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. pr_auc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the precision recall curve — pr_auc","text":"Macro macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the precision recall curve — pr_auc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the precision recall curve — pr_auc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_auc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the precision recall curve — pr_auc","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. pr_auc(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc binary 0.946 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% pr_auc(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc macro 0.611 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% pr_auc(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 pr_auc macro 0.611 # Groups are respected hpc_cv %>% group_by(Resample) %>% pr_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 pr_auc macro 0.611 #> 2 Fold02 pr_auc macro 0.620 #> 3 Fold03 pr_auc macro 0.689 #> 4 Fold04 pr_auc macro 0.680 #> 5 Fold05 pr_auc macro 0.620 #> 6 Fold06 pr_auc macro 0.650 #> 7 Fold07 pr_auc macro 0.607 #> 8 Fold08 pr_auc macro 0.650 #> 9 Fold09 pr_auc macro 0.628 #> 10 Fold10 pr_auc macro 0.603 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% pr_auc(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 pr_auc macro_weighted 0.746 #> 2 Fold02 pr_auc macro_weighted 0.743 #> 3 Fold03 pr_auc macro_weighted 0.789 #> 4 Fold04 pr_auc macro_weighted 0.754 #> 5 Fold05 pr_auc macro_weighted 0.737 #> 6 Fold06 pr_auc macro_weighted 0.743 #> 7 Fold07 pr_auc macro_weighted 0.748 #> 8 Fold08 pr_auc macro_weighted 0.756 #> 9 Fold09 pr_auc macro_weighted 0.711 #> 10 Fold10 pr_auc macro_weighted 0.737 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") pr_auc_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.6109931"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Precision recall curve — pr_curve","title":"Precision recall curve — pr_curve","text":"pr_curve() constructs full precision recall curve returns tibble. See pr_auc() area precision recall curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Precision recall curve — pr_curve","text":"","code":"pr_curve(data, ...) # S3 method for data.frame pr_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL )"},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Precision recall curve — pr_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Precision recall curve — pr_curve","text":"tibble class pr_df pr_grouped_df columns .threshold, recall, precision.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Precision recall curve — pr_curve","text":"pr_curve() computes precision every unique value probability column (addition infinity). ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Precision recall curve — pr_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Precision recall curve — pr_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Precision recall curve — pr_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/pr_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Precision recall curve — pr_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. pr_curve(two_class_example, truth, Class1) #> # A tibble: 501 × 3 #> .threshold recall precision #> #> 1 Inf 0 1 #> 2 1.00 0.00388 1 #> 3 1.00 0.00775 1 #> 4 1.00 0.0116 1 #> 5 1.00 0.0155 1 #> 6 1.00 0.0194 1 #> 7 1.00 0.0233 1 #> 8 1.00 0.0271 1 #> 9 1.00 0.0310 1 #> 10 1.00 0.0349 1 #> # ℹ 491 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) pr_curve(two_class_example, truth, Class1) %>% ggplot(aes(x = recall, y = precision)) + geom_path() + coord_equal() + theme_bw() # Or use autoplot autoplot(pr_curve(two_class_example, truth, Class1)) # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% pr_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% pr_curve(obs, VF:L) %>% autoplot()"},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":null,"dir":"Reference","previous_headings":"","what":"Precision — precision","title":"Precision — precision","text":"functions calculate precision() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions recall() f_meas().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Precision — precision","text":"","code":"precision(data, ...) # S3 method for data.frame precision( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) precision_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Precision — precision","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Precision — precision","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. precision_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Precision — precision","text":"precision percentage predicted truly relevant results total number predicted relevant results characterizes \"purity retrieval performance\" (Buckland Gey, 1994). denominator calculation 0, precision undefined. happens # true_positive = 0 # false_positive = 0 true, mean predicted events. computing binary precision, NA value returned warning. computing multiclass precision, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Precision — precision","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Precision — precision","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Precision — precision","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Precision — precision","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Precision — precision","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/precision.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Precision — precision","text":"","code":"# Two class data(\"two_class_example\") precision(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision binary 0.819 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% precision(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 precision macro 0.637 # Groups are respected hpc_cv %>% group_by(Resample) %>% precision(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 precision macro 0.637 #> 2 Fold02 precision macro 0.603 #> 3 Fold03 precision macro 0.706 #> 4 Fold04 precision macro 0.658 #> 5 Fold05 precision macro 0.651 #> 6 Fold06 precision macro 0.626 #> 7 Fold07 precision macro 0.562 #> 8 Fold08 precision macro 0.652 #> 9 Fold09 precision macro 0.605 #> 10 Fold10 precision macro 0.625 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% precision(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 precision macro_weighted 0.697 #> 2 Fold02 precision macro_weighted 0.690 #> 3 Fold03 precision macro_weighted 0.752 #> 4 Fold04 precision macro_weighted 0.690 #> 5 Fold05 precision macro_weighted 0.705 #> 6 Fold06 precision macro_weighted 0.682 #> 7 Fold07 precision macro_weighted 0.649 #> 8 Fold08 precision macro_weighted 0.702 #> 9 Fold09 precision macro_weighted 0.661 #> 10 Fold10 precision macro_weighted 0.683 # Vector version precision_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.8194946 # Making Class2 the \"relevant\" level precision_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.8609865"},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":null,"dir":"Reference","previous_headings":"","what":"Recall — recall","title":"Recall — recall","text":"functions calculate recall() measurement system finding relevant documents compared reference results (truth regarding relevance). Highly related functions precision() f_meas().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Recall — recall","text":"","code":"recall(data, ...) # S3 method for data.frame recall( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) recall_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Recall — recall","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Recall — recall","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. recall_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Recall — recall","text":"recall (aka sensitivity) defined proportion relevant results number samples actually relevant. relevant results, recall defined value NA returned. denominator calculation 0, recall undefined. happens # true_positive = 0 # false_negative = 0 true, mean true events. computing binary recall, NA value returned warning. computing multiclass recall, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Recall — recall","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Recall — recall","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Recall — recall","text":"Suppose 2x2 table notation: formulas used : $$recall = /(+C)$$ $$precision = /(+B)$$ $$F_{meas} = (1+\\beta^2) * precision * recall/((\\beta^2 * precision)+recall)$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Recall — recall","text":"Buckland, M., & Gey, F. (1994). relationship Recall Precision. Journal American Society Information Science, 45(1), 12-19. Powers, D. (2007). Evaluation: Precision, Recall F Factor ROC, Informedness, Markedness Correlation. Technical Report SIE-07-001, Flinders University","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Recall — recall","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/recall.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Recall — recall","text":"","code":"# Two class data(\"two_class_example\") recall(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 recall binary 0.880 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% recall(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 recall macro 0.548 # Groups are respected hpc_cv %>% group_by(Resample) %>% recall(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 recall macro 0.548 #> 2 Fold02 recall macro 0.541 #> 3 Fold03 recall macro 0.634 #> 4 Fold04 recall macro 0.570 #> 5 Fold05 recall macro 0.550 #> 6 Fold06 recall macro 0.540 #> 7 Fold07 recall macro 0.531 #> 8 Fold08 recall macro 0.584 #> 9 Fold09 recall macro 0.568 #> 10 Fold10 recall macro 0.537 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% recall(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 recall macro_weighted 0.726 #> 2 Fold02 recall macro_weighted 0.712 #> 3 Fold03 recall macro_weighted 0.758 #> 4 Fold04 recall macro_weighted 0.712 #> 5 Fold05 recall macro_weighted 0.712 #> 6 Fold06 recall macro_weighted 0.697 #> 7 Fold07 recall macro_weighted 0.675 #> 8 Fold08 recall macro_weighted 0.721 #> 9 Fold09 recall macro_weighted 0.673 #> 10 Fold10 recall macro_weighted 0.699 # Vector version recall_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.879845 # Making Class2 the \"relevant\" level recall_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.7933884"},{"path":"https://yardstick.tidymodels.org/dev/reference/reexports.html","id":null,"dir":"Reference","previous_headings":"","what":"Objects exported from other packages — reexports","title":"Objects exported from other packages — reexports","text":"objects imported packages. Follow links see documentation. generics tidy","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":null,"dir":"Reference","previous_headings":"","what":"Root mean squared error — rmse","title":"Root mean squared error — rmse","text":"Calculate root mean squared error. rmse() metric units original data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Root mean squared error — rmse","text":"","code":"rmse(data, ...) # S3 method for data.frame rmse(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rmse_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Root mean squared error — rmse","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Root mean squared error — rmse","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rmse_vec(), single numeric value (NA).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Root mean squared error — rmse","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rmse.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Root mean squared error — rmse","text":"","code":"# Supply truth and predictions as bare column names rmse(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rmse standard 0.722 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rmse(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rmse standard 0.715 #> 2 10 rmse standard 0.673 #> 3 2 rmse standard 0.716 #> 4 3 rmse standard 0.644 #> 5 4 rmse standard 0.737 #> 6 5 rmse standard 0.675 #> 7 6 rmse standard 0.807 #> 8 7 rmse standard 0.801 #> 9 8 rmse standard 0.635 #> 10 9 rmse standard 0.692 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.710"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the receiver operator curve — roc_auc","title":"Area under the receiver operator curve — roc_auc","text":"roc_auc() metric computes area ROC curve. See roc_curve() full curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the receiver operator curve — roc_auc","text":"","code":"roc_auc(data, ...) # S3 method for data.frame roc_auc( data, truth, ..., estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list() ) roc_auc_vec( truth, estimate, estimator = NULL, na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the receiver operator curve — roc_auc","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimator One \"binary\", \"hand_till\", \"macro\", \"macro_weighted\" specify type averaging done. \"binary\" relevant two class case. others general methods calculating multiclass metrics. default automatically choose \"binary\" truth binary, \"hand_till\" truth >2 levels case_weights specified, \"macro\" truth >2 levels case_weights specified (case \"hand_till\" well-defined). na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the receiver operator curve — roc_auc","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_auc_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Area under the receiver operator curve — roc_auc","text":"Generally, ROC AUC value 0.5 1, 1 perfect prediction model. value 0 0.5, implies meaningful information model, applied incorrectly opposite model predicts result AUC >0.5. Note combine estimator = \"hand_till\" case_weights.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the receiver operator curve — roc_auc","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the receiver operator curve — roc_auc","text":"default multiclass method computing roc_auc() use method Hand, Till, (2001). Unlike macro-averaging, method insensitive class distributions like binary ROC AUC case. Additionally, multiclass techniques return NA levels truth occur zero times actual data, Hand-Till method simply ignore levels averaging calculation, warning. Macro macro-weighted averaging still provided, even though default. fact, macro-weighted averaging corresponds definition multiclass AUC given Provost Domingos (2001).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the receiver operator curve — roc_auc","text":"Hand, Till (2001). \"Simple Generalisation Area ROC Curve Multiple Class Classification Problems\". Machine Learning. Vol 45, Iss 2, pp 171-186. Fawcett (2005). \"introduction ROC analysis\". Pattern Recognition Letters. 27 (2006), pp 861-874. Provost, F., Domingos, P., 2001. \"Well-trained PETs: Improving probability estimation trees\", CeDER Working Paper #-00-04, Stern School Business, New York University, NY, NY 10012.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the receiver operator curve — roc_auc","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the receiver operator curve — roc_auc","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. roc_auc(two_class_example, truth, Class1) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc binary 0.939 # --------------------------------------------------------------------------- # Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_auc(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc hand_till 0.813 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_auc(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_auc hand_till 0.813 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc hand_till 0.813 #> 2 Fold02 roc_auc hand_till 0.817 #> 3 Fold03 roc_auc hand_till 0.869 #> 4 Fold04 roc_auc hand_till 0.849 #> 5 Fold05 roc_auc hand_till 0.811 #> 6 Fold06 roc_auc hand_till 0.836 #> 7 Fold07 roc_auc hand_till 0.825 #> 8 Fold08 roc_auc hand_till 0.846 #> 9 Fold09 roc_auc hand_till 0.828 #> 10 Fold10 roc_auc hand_till 0.812 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% roc_auc(obs, VF:L, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_auc macro_weighted 0.880 #> 2 Fold02 roc_auc macro_weighted 0.873 #> 3 Fold03 roc_auc macro_weighted 0.906 #> 4 Fold04 roc_auc macro_weighted 0.867 #> 5 Fold05 roc_auc macro_weighted 0.866 #> 6 Fold06 roc_auc macro_weighted 0.865 #> 7 Fold07 roc_auc macro_weighted 0.868 #> 8 Fold08 roc_auc macro_weighted 0.865 #> 9 Fold09 roc_auc macro_weighted 0.841 #> 10 Fold10 roc_auc macro_weighted 0.869 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_auc_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8131924"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Compute area ROC survival curve using predicted survival probabilities corresponds different time points.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"","code":"roc_auc_survival(data, ...) # S3 method for data.frame roc_auc_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL) roc_auc_survival_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. estimate truth binary, numeric vector class probabilities corresponding \"relevant\" class. Otherwise, matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"tibble columns .metric, .estimator, .estimate. ungrouped data frame, result one row values. grouped data frame, number rows returned number groups. roc_auc_survival_vec(), numeric vector length input argument eval_time. (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes area ROC curve. account censoring, inverse probability censoring weights (IPCW) used calculations. See equation 7 section 4.3 Blanche al (2013) details. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument. Larger values score associated better model performance.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Blanche, P., Dartigues, J.-F. Jacqmin-Gadda, H. (2013), Review comparison ROC curve estimators time-dependent outcome marker-dependent censoring. Biom. J., 55: 687-704. Graf, E., Schmoor, C., Sauerbrei, W. Schumacher, M. (1999), Assessment comparison prognostic classification schemes survival data. Statist. Med., 18: 2529-2545.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_auc_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent ROC AUC for Censored Data — roc_auc_survival","text":"","code":"library(dplyr) lung_surv %>% roc_auc_survival( truth = surv_obj, .pred ) #> # A tibble: 5 × 4 #> .metric .estimator .eval_time .estimate #> #> 1 roc_auc_survival standard 100 0.659 #> 2 roc_auc_survival standard 200 0.679 #> 3 roc_auc_survival standard 300 0.688 #> 4 roc_auc_survival standard 400 0.648 #> 5 roc_auc_survival standard 500 0.662"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"roc_aunp() multiclass metric computes area ROC curve class rest, using priori class distribution. equivalent roc_auc(estimator = \"macro_weighted\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"","code":"roc_aunp(data, ...) # S3 method for data.frame roc_aunp(data, truth, ..., na_rm = TRUE, case_weights = NULL, options = list()) roc_aunp_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. many columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_aunp_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"multiclass method computing area ROC curve uses priori class distribution equivalent roc_auc(estimator = \"macro_weighted\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"Ferri, C., Hernández-Orallo, J., & Modroiu, R. (2009). \"experimental comparison performance measures classification\". Pattern Recognition Letters. 30 (1), pp 27-38.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"Julia Silge","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunp.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the ROC curve of each class against the rest, using the a priori\nclass distribution — roc_aunp","text":"","code":"# Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_aunp(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunp macro 0.880 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_aunp(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunp macro 0.880 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_aunp(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_aunp macro 0.880 #> 2 Fold02 roc_aunp macro 0.873 #> 3 Fold03 roc_aunp macro 0.906 #> 4 Fold04 roc_aunp macro 0.867 #> 5 Fold05 roc_aunp macro 0.866 #> 6 Fold06 roc_aunp macro 0.865 #> 7 Fold07 roc_aunp macro 0.868 #> 8 Fold08 roc_aunp macro 0.865 #> 9 Fold09 roc_aunp macro 0.841 #> 10 Fold10 roc_aunp macro 0.869 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_aunp_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8795121"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":null,"dir":"Reference","previous_headings":"","what":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"roc_aunu() multiclass metric computes area ROC curve class rest, using uniform class distribution. equivalent roc_auc(estimator = \"macro\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"","code":"roc_aunu(data, ...) # S3 method for data.frame roc_aunu(data, truth, ..., na_rm = TRUE, case_weights = NULL, options = list()) roc_aunu_vec( truth, estimate, na_rm = TRUE, case_weights = NULL, options = list(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. many columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly. estimate matrix many columns factor levels truth. assumed order levels truth.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. roc_aunu_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"multiclass method computing area ROC curve uses uniform class distribution equivalent roc_auc(estimator = \"macro\").","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"Ferri, C., Hernández-Orallo, J., & Modroiu, R. (2009). \"experimental comparison performance measures classification\". Pattern Recognition Letters. 30 (1), pp 27-38.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"Julia Silge","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_aunu.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Area under the ROC curve of each class against the rest, using the uniform\nclass distribution — roc_aunu","text":"","code":"# Multiclass example # `obs` is a 4 level factor. The first level is `\"VF\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(hpc_cv) # You can use the col1:colN tidyselect syntax library(dplyr) hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_aunu(obs, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunu macro 0.871 # Change the first level of `obs` from `\"VF\"` to `\"M\"` to alter the # event of interest. The class probability columns should be supplied # in the same order as the levels. hpc_cv %>% filter(Resample == \"Fold01\") %>% mutate(obs = relevel(obs, \"M\")) %>% roc_aunu(obs, M, VF:L) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 roc_aunu macro 0.871 # Groups are respected hpc_cv %>% group_by(Resample) %>% roc_aunu(obs, VF:L) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 roc_aunu macro 0.871 #> 2 Fold02 roc_aunu macro 0.863 #> 3 Fold03 roc_aunu macro 0.898 #> 4 Fold04 roc_aunu macro 0.874 #> 5 Fold05 roc_aunu macro 0.865 #> 6 Fold06 roc_aunu macro 0.877 #> 7 Fold07 roc_aunu macro 0.865 #> 8 Fold08 roc_aunu macro 0.873 #> 9 Fold09 roc_aunu macro 0.855 #> 10 Fold10 roc_aunu macro 0.865 # Vector version # Supply a matrix of class probabilities fold1 <- hpc_cv %>% filter(Resample == \"Fold01\") roc_aunu_vec( truth = fold1$obs, matrix( c(fold1$VF, fold1$F, fold1$M, fold1$L), ncol = 4 ) ) #> [1] 0.8714461"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":null,"dir":"Reference","previous_headings":"","what":"Receiver operator curve — roc_curve","title":"Receiver operator curve — roc_curve","text":"roc_curve() constructs full ROC curve returns tibble. See roc_auc() area ROC curve.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Receiver operator curve — roc_curve","text":"","code":"roc_curve(data, ...) # S3 method for data.frame roc_curve( data, truth, ..., na_rm = TRUE, event_level = yardstick_event_level(), case_weights = NULL, options = list() )"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Receiver operator curve — roc_curve","text":"data data.frame containing columns specified truth .... ... set unquoted column names one dplyr selector functions choose variables contain class probabilities. truth binary, 1 column selected, correspond value event_level. Otherwise, many columns factor levels truth ordering columns factor levels truth. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. na_rm logical value indicating whether NA values stripped computation proceeds. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. options [deprecated] longer supported yardstick 1.0.0. pass something ignored warning. Previously, options passed pROC::roc(). need support , use pROC package directly.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Receiver operator curve — roc_curve","text":"tibble class roc_df roc_grouped_df columns .threshold, specificity, sensitivity.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Receiver operator curve — roc_curve","text":"roc_curve() computes sensitivity every unique value probability column (addition infinity minus infinity). ggplot2::autoplot() method quickly visualizing curve. works binary multiclass output, also works grouped data (.e. resamples). See examples.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Receiver operator curve — roc_curve","text":"multiclass truth column provided, one-vs-approach taken calculate multiple curves, one per level. case, additional column, .level, identifying \"one\" column one-vs-calculation.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Receiver operator curve — roc_curve","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Receiver operator curve — roc_curve","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Receiver operator curve — roc_curve","text":"","code":"# --------------------------------------------------------------------------- # Two class example # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section above. data(two_class_example) # Binary metrics using class probabilities take a factor `truth` column, # and a single class probability column containing the probabilities of # the event of interest. Here, since `\"Class1\"` is the first level of # `\"truth\"`, it is the event of interest and we pass in probabilities for it. roc_curve(two_class_example, truth, Class1) #> # A tibble: 502 × 3 #> .threshold specificity sensitivity #> #> 1 -Inf 0 1 #> 2 1.79e-7 0 1 #> 3 4.50e-6 0.00413 1 #> 4 5.81e-6 0.00826 1 #> 5 5.92e-6 0.0124 1 #> 6 1.22e-5 0.0165 1 #> 7 1.40e-5 0.0207 1 #> 8 1.43e-5 0.0248 1 #> 9 2.38e-5 0.0289 1 #> 10 3.30e-5 0.0331 1 #> # ℹ 492 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) roc_curve(two_class_example, truth, Class1) %>% ggplot(aes(x = 1 - specificity, y = sensitivity)) + geom_path() + geom_abline(lty = 3) + coord_equal() + theme_bw() # Or use autoplot autoplot(roc_curve(two_class_example, truth, Class1)) if (FALSE) { # Multiclass one-vs-all approach # One curve per level hpc_cv %>% filter(Resample == \"Fold01\") %>% roc_curve(obs, VF:L) %>% autoplot() # Same as above, but will all of the resamples hpc_cv %>% group_by(Resample) %>% roc_curve(obs, VF:L) %>% autoplot() }"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":null,"dir":"Reference","previous_headings":"","what":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Compute ROC survival curve using predicted survival probabilities corresponds different time points.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"","code":"roc_curve_survival(data, ...) # S3 method for data.frame roc_curve_survival(data, truth, ..., na_rm = TRUE, case_weights = NULL)"},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"data data.frame containing columns specified truth .... ... column identifier survival probabilities list column data.frames corresponding output given predicting censored model. unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. See Details . truth column identifier true survival result (created using survival::Surv().). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, survival::Surv() object. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"tibble class roc_survival_df, grouped_roc_survival_df columns .threshold, sensitivity, specificity, .eval_time.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"formulation takes survival probability predictions one specific evaluation times , time, computes ROC curve. account censoring, inverse probability censoring weights (IPCW) used calculations. See equation 7 section 4.3 Blanche al (2013) details. column passed ... list column one element per independent experiential unit (e.g. patient). list column contain data frames several columns: .eval_time: time prediction made. .pred_survival: predicted probability survival .eval_time .weight_censored: case weight inverse probability censoring. last column can produced using parsnip::.censoring_weights_graf(). corresponds weighting scheme Graf et al (1999). internal data set lung_surv shows example format. method automatically groups .eval_time argument.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Blanche, P., Dartigues, J.-F. Jacqmin-Gadda, H. (2013), Review comparison ROC curve estimators time-dependent outcome marker-dependent censoring. Biom. J., 55: 687-704. Graf, E., Schmoor, C., Sauerbrei, W. Schumacher, M. (1999), Assessment comparison prognostic classification schemes survival data. Statist. Med., 18: 2529-2545.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"Emil Hvitfeldt","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/roc_curve_survival.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Time-Dependent ROC surve for Censored Data — roc_curve_survival","text":"","code":"result <- roc_curve_survival( lung_surv, truth = surv_obj, .pred ) result #> # A tibble: 650 × 4 #> .threshold sensitivity specificity .eval_time #> #> 1 -Inf 0 1 100 #> 2 0.615 0 0.995 100 #> 3 0.723 0.0334 0.985 100 #> 4 0.727 0.133 0.985 100 #> 5 0.730 0.167 0.985 100 #> 6 0.732 0.233 0.980 100 #> 7 0.739 0.233 0.969 100 #> 8 0.741 0.267 0.964 100 #> 9 0.746 0.267 0.959 100 #> 10 0.748 0.267 0.954 100 #> # ℹ 640 more rows # --------------------------------------------------------------------------- # `autoplot()` # Visualize the curve using ggplot2 manually library(ggplot2) library(dplyr) result %>% mutate(.eval_time = format(.eval_time)) %>% ggplot(aes( x = 1 - specificity, y = sensitivity, group = .eval_time, col = .eval_time )) + geom_step(direction = \"hv\") + geom_abline(lty = 3) + coord_equal() + theme_bw() # Or use autoplot autoplot(result)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":null,"dir":"Reference","previous_headings":"","what":"Ratio of performance to deviation — rpd","title":"Ratio of performance to deviation — rpd","text":"functions appropriate cases model outcome numeric. ratio performance deviation (rpd()) ratio performance inter-quartile (rpiq()) measures consistency/correlation observed predicted values (accuracy).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ratio of performance to deviation — rpd","text":"","code":"rpd(data, ...) # S3 method for data.frame rpd(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rpd_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ratio of performance to deviation — rpd","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ratio of performance to deviation — rpd","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rpd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Ratio of performance to deviation — rpd","text":"field spectroscopy particular, ratio performance deviation (RPD) used standard way report quality model. ratio standard deviation variable standard error prediction variable given model. However, systematic use criticized several authors, since using standard deviation represent spread variable can misleading skewed dataset. ratio performance inter-quartile introduced Bellon-Maurel et al. (2010) address issues, generalise RPD non-normally distributed variables.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ratio of performance to deviation — rpd","text":"Williams, P.C. (1987) Variables affecting near-infrared reflectance spectroscopic analysis. : Near Infrared Technology Agriculture Food Industries. 1st Ed. P.Williams K.Norris, Eds. . Cereal Assoc. Cereal Chem., St. Paul, MN. Bellon-Maurel, V., Fernandez-Ahumada, E., Palagos, B., Roger, J.M. McBratney, ., (2010). Critical review chemometric indicators commonly used assessing quality prediction soil attributes NIR spectroscopy. TrAC Trends Analytical Chemistry, 29(9), pp.1073-1081.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Ratio of performance to deviation — rpd","text":"Pierre Roudier","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpd.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ratio of performance to deviation — rpd","text":"","code":"# Supply truth and predictions as bare column names rpd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rpd standard 2.88 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rpd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rpd standard 2.78 #> 2 10 rpd standard 2.87 #> 3 2 rpd standard 3.04 #> 4 3 rpd standard 3.41 #> 5 4 rpd standard 3.02 #> 6 5 rpd standard 2.66 #> 7 6 rpd standard 2.81 #> 8 7 rpd standard 2.61 #> 9 8 rpd standard 3.45 #> 10 9 rpd standard 2.93 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 2.96"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":null,"dir":"Reference","previous_headings":"","what":"Ratio of performance to inter-quartile — rpiq","title":"Ratio of performance to inter-quartile — rpiq","text":"functions appropriate cases model outcome numeric. ratio performance deviation (rpd()) ratio performance inter-quartile (rpiq()) measures consistency/correlation observed predicted values (accuracy).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Ratio of performance to inter-quartile — rpiq","text":"","code":"rpiq(data, ...) # S3 method for data.frame rpiq(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rpiq_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Ratio of performance to inter-quartile — rpiq","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Ratio of performance to inter-quartile — rpiq","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rpd_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Ratio of performance to inter-quartile — rpiq","text":"field spectroscopy particular, ratio performance deviation (RPD) used standard way report quality model. ratio standard deviation variable standard error prediction variable given model. However, systematic use criticized several authors, since using standard deviation represent spread variable can misleading skewed dataset. ratio performance inter-quartile introduced Bellon-Maurel et al. (2010) address issues, generalise RPD non-normally distributed variables.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Ratio of performance to inter-quartile — rpiq","text":"Williams, P.C. (1987) Variables affecting near-infrared reflectance spectroscopic analysis. : Near Infrared Technology Agriculture Food Industries. 1st Ed. P.Williams K.Norris, Eds. . Cereal Assoc. Cereal Chem., St. Paul, MN. Bellon-Maurel, V., Fernandez-Ahumada, E., Palagos, B., Roger, J.M. McBratney, ., (2010). Critical review chemometric indicators commonly used assessing quality prediction soil attributes NIR spectroscopy. TrAC Trends Analytical Chemistry, 29(9), pp.1073-1081.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Ratio of performance to inter-quartile — rpiq","text":"Pierre Roudier","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rpiq.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Ratio of performance to inter-quartile — rpiq","text":"","code":"# Supply truth and predictions as bare column names rpd(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rpd standard 2.88 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rpd(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rpd standard 2.78 #> 2 10 rpd standard 2.87 #> 3 2 rpd standard 3.04 #> 4 3 rpd standard 3.41 #> 5 4 rpd standard 3.02 #> 6 5 rpd standard 2.66 #> 7 6 rpd standard 2.81 #> 8 7 rpd standard 2.61 #> 9 8 rpd standard 3.45 #> 10 9 rpd standard 2.93 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 2.96"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":null,"dir":"Reference","previous_headings":"","what":"R squared — rsq","title":"R squared — rsq","text":"Calculate coefficient determination using correlation. traditional measure R squared, see rsq_trad().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"R squared — rsq","text":"","code":"rsq(data, ...) # S3 method for data.frame rsq(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rsq_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"R squared — rsq","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"R squared — rsq","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rsq_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"R squared — rsq","text":"two estimates coefficient determination, rsq() rsq_trad(), differ formula. former guarantees value (0, 1) latter can generate inaccurate values model non-informative (see examples). measures consistency/correlation accuracy. rsq() simply squared correlation truth estimate. rsq() internally computes correlation, either truth estimate constant can result divide zero error. cases, warning thrown NA returned. can occur model predicts single value samples. example, regularized model eliminates predictors except intercept . Another example CART model contains splits.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"R squared — rsq","text":"Kvalseth. Cautionary note \\(R^2\\). American Statistician (1985) vol. 39 (4) pp. 279-285.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"R squared — rsq","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"R squared — rsq","text":"","code":"# Supply truth and predictions as bare column names rsq(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.879 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rsq(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rsq standard 0.874 #> 2 10 rsq standard 0.879 #> 3 2 rsq standard 0.891 #> 4 3 rsq standard 0.916 #> 5 4 rsq standard 0.892 #> 6 5 rsq standard 0.858 #> 7 6 rsq standard 0.873 #> 8 7 rsq standard 0.852 #> 9 8 rsq standard 0.915 #> 10 9 rsq standard 0.884 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.883 # With uninformitive data, the traditional version of R^2 can return # negative values. set.seed(2291) solubility_test$randomized <- sample(solubility_test$prediction) rsq(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.00199 rsq_trad(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard -1.01 # A constant `truth` or `estimate` vector results in a warning from # a divide by zero error in the correlation calculation. # `NA` will be returned in these cases. truth <- c(1, 2) estimate <- c(1, 1) rsq_vec(truth, estimate) #> Warning: A correlation computation is required, but `estimate` is constant and has #> 0 standard deviation, resulting in a divide by 0 error. `NA` will be #> returned. #> [1] NA"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":null,"dir":"Reference","previous_headings":"","what":"R squared - traditional — rsq_trad","title":"R squared - traditional — rsq_trad","text":"Calculate coefficient determination using traditional definition R squared using sum squares. measure R squared strictly (0, 1), see rsq().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"R squared - traditional — rsq_trad","text":"","code":"rsq_trad(data, ...) # S3 method for data.frame rsq_trad(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) rsq_trad_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"R squared - traditional — rsq_trad","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"R squared - traditional — rsq_trad","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. rsq_trad_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"R squared - traditional — rsq_trad","text":"two estimates coefficient determination, rsq() rsq_trad(), differ formula. former guarantees value (0, 1) latter can generate inaccurate values model non-informative (see examples). measures consistency/correlation accuracy.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"R squared - traditional — rsq_trad","text":"Kvalseth. Cautionary note \\(R^2\\). American Statistician (1985) vol. 39 (4) pp. 279-285.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"R squared - traditional — rsq_trad","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/rsq_trad.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"R squared - traditional — rsq_trad","text":"","code":"# Supply truth and predictions as bare column names rsq_trad(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard 0.879 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% rsq_trad(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 rsq_trad standard 0.870 #> 2 10 rsq_trad standard 0.878 #> 3 2 rsq_trad standard 0.891 #> 4 3 rsq_trad standard 0.913 #> 5 4 rsq_trad standard 0.889 #> 6 5 rsq_trad standard 0.857 #> 7 6 rsq_trad standard 0.872 #> 8 7 rsq_trad standard 0.852 #> 9 8 rsq_trad standard 0.915 #> 10 9 rsq_trad standard 0.883 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 0.882 # With uninformitive data, the traditional version of R^2 can return # negative values. set.seed(2291) solubility_test$randomized <- sample(solubility_test$prediction) rsq(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq standard 0.00199 rsq_trad(solubility_test, solubility, randomized) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 rsq_trad standard -1.01"},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":null,"dir":"Reference","previous_headings":"","what":"Sensitivity — sens","title":"Sensitivity — sens","text":"functions calculate sens() (sensitivity) measurement system compared reference result (\"truth\" gold standard). Highly related functions spec(), ppv(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Sensitivity — sens","text":"","code":"sens(data, ...) # S3 method for data.frame sens( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sens_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sensitivity(data, ...) # S3 method for data.frame sensitivity( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) sensitivity_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Sensitivity — sens","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Sensitivity — sens","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. sens_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Sensitivity — sens","text":"sensitivity (sens()) defined proportion positive results number samples actually positive. denominator calculation 0, sensitivity undefined. happens # true_positive = 0 # false_negative = 0 true, mean true events. computing binary sensitivity, NA value returned warning. computing multiclass sensitivity, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Sensitivity — sens","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Sensitivity — sens","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Sensitivity — sens","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Sensitivity — sens","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Sensitivity — sens","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/sens.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Sensitivity — sens","text":"","code":"# Two class data(\"two_class_example\") sens(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 sens binary 0.880 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% sens(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 sens macro 0.548 # Groups are respected hpc_cv %>% group_by(Resample) %>% sens(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 sens macro 0.548 #> 2 Fold02 sens macro 0.541 #> 3 Fold03 sens macro 0.634 #> 4 Fold04 sens macro 0.570 #> 5 Fold05 sens macro 0.550 #> 6 Fold06 sens macro 0.540 #> 7 Fold07 sens macro 0.531 #> 8 Fold08 sens macro 0.584 #> 9 Fold09 sens macro 0.568 #> 10 Fold10 sens macro 0.537 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% sens(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 sens macro_weighted 0.726 #> 2 Fold02 sens macro_weighted 0.712 #> 3 Fold03 sens macro_weighted 0.758 #> 4 Fold04 sens macro_weighted 0.712 #> 5 Fold05 sens macro_weighted 0.712 #> 6 Fold06 sens macro_weighted 0.697 #> 7 Fold07 sens macro_weighted 0.675 #> 8 Fold08 sens macro_weighted 0.721 #> 9 Fold09 sens macro_weighted 0.673 #> 10 Fold10 sens macro_weighted 0.699 # Vector version sens_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.879845 # Making Class2 the \"relevant\" level sens_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.7933884"},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":null,"dir":"Reference","previous_headings":"","what":"Symmetric mean absolute percentage error — smape","title":"Symmetric mean absolute percentage error — smape","text":"Calculate symmetric mean absolute percentage error. metric relative units.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Symmetric mean absolute percentage error — smape","text":"","code":"smape(data, ...) # S3 method for data.frame smape(data, truth, estimate, na_rm = TRUE, case_weights = NULL, ...) smape_vec(truth, estimate, na_rm = TRUE, case_weights = NULL, ...)"},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Symmetric mean absolute percentage error — smape","text":"data data.frame containing columns specified truth estimate arguments. ... currently used. truth column identifier true results (numeric). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, numeric vector. estimate column identifier predicted results (also numeric). truth can specified different ways primary method use unquoted variable name. _vec() functions, numeric vector. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Symmetric mean absolute percentage error — smape","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. smape_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Symmetric mean absolute percentage error — smape","text":"implementation smape() \"usual definition\" denominator divided two.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Symmetric mean absolute percentage error — smape","text":"Max Kuhn, Riaz Hedayati","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/smape.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Symmetric mean absolute percentage error — smape","text":"","code":"# Supply truth and predictions as bare column names smape(solubility_test, solubility, prediction) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 smape standard 36.7 library(dplyr) set.seed(1234) size <- 100 times <- 10 # create 10 resamples solubility_resampled <- bind_rows( replicate( n = times, expr = sample_n(solubility_test, size, replace = TRUE), simplify = FALSE ), .id = \"resample\" ) # Compute the metric by group metric_results <- solubility_resampled %>% group_by(resample) %>% smape(solubility, prediction) metric_results #> # A tibble: 10 × 4 #> resample .metric .estimator .estimate #> #> 1 1 smape standard 33.5 #> 2 10 smape standard 28.9 #> 3 2 smape standard 41.6 #> 4 3 smape standard 28.4 #> 5 4 smape standard 37.0 #> 6 5 smape standard 31.8 #> 7 6 smape standard 48.3 #> 8 7 smape standard 27.8 #> 9 8 smape standard 35.6 #> 10 9 smape standard 30.2 # Resampled mean estimate metric_results %>% summarise(avg_estimate = mean(.estimate)) #> # A tibble: 1 × 1 #> avg_estimate #> #> 1 34.3"},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":null,"dir":"Reference","previous_headings":"","what":"Solubility Predictions from MARS Model — solubility_test","title":"Solubility Predictions from MARS Model — solubility_test","text":"Solubility Predictions MARS Model","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"source","dir":"Reference","previous_headings":"","what":"Source","title":"Solubility Predictions from MARS Model — solubility_test","text":"Kuhn, M., Johnson, K. (2013) Applied Predictive Modeling, Springer","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Solubility Predictions from MARS Model — solubility_test","text":"solubility_test data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Solubility Predictions from MARS Model — solubility_test","text":"solubility data Kuhn Johnson (2013), data test set results MARS model. observed solubility (column solubility) model results (prediction) contained data.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/solubility_test.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Solubility Predictions from MARS Model — solubility_test","text":"","code":"data(solubility_test) str(solubility_test) #> 'data.frame':\t316 obs. of 2 variables: #> $ solubility: num 0.93 0.85 0.81 0.74 0.61 0.58 0.57 0.56 0.52 0.45 ... #> $ prediction: num 0.368 -0.15 -0.505 0.54 -0.479 ..."},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":null,"dir":"Reference","previous_headings":"","what":"Specificity — spec","title":"Specificity — spec","text":"functions calculate spec() (specificity) measurement system compared reference result (\"truth\" gold standard). Highly related functions sens(), ppv(), npv().","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Specificity — spec","text":"","code":"spec(data, ...) # S3 method for data.frame spec( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) spec_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) specificity(data, ...) # S3 method for data.frame specificity( data, truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... ) specificity_vec( truth, estimate, estimator = NULL, na_rm = TRUE, case_weights = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Specificity — spec","text":"data Either data.frame containing columns specified truth estimate arguments, table/matrix true class results columns table. ... currently used. truth column identifier true class results (factor). unquoted column name although argument passed expression supports quasiquotation (can unquote column names). _vec() functions, factor vector. estimate column identifier predicted class results (also factor). truth can specified different ways primary method use unquoted variable name. _vec() functions, factor vector. estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. na_rm logical value indicating whether NA values stripped computation proceeds. case_weights optional column identifier case weights. unquoted column name evaluates numeric column data. _vec() functions, numeric vector. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Specificity — spec","text":"tibble columns .metric, .estimator, .estimate 1 row values. grouped data frames, number rows returned number groups. spec_vec(), single numeric value (NA).","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Specificity — spec","text":"specificity measures proportion negatives correctly identified negatives. denominator calculation 0, specificity undefined. happens # true_negative = 0 # false_positive = 0 true, mean true negatives. computing binary specificity, NA value returned warning. computing multiclass specificity, individual NA values removed, computation procede, warning.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Specificity — spec","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"multiclass","dir":"Reference","previous_headings":"","what":"Multiclass","title":"Specificity — spec","text":"Macro, micro, macro-weighted averaging available metric. default select macro averaging truth factor 2 levels provided. Otherwise, standard binary calculation done. See vignette(\"multiclass\", \"yardstick\") information.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"implementation","dir":"Reference","previous_headings":"","what":"Implementation","title":"Specificity — spec","text":"Suppose 2x2 table notation: formulas used : $$Sensitivity = /(+C)$$ $$Specificity = D/(B+D)$$ $$Prevalence = (+C)/(+B+C+D)$$ $$PPV = (Sensitivity * Prevalence) / ((Sensitivity * Prevalence) + ((1-Specificity) * (1-Prevalence)))$$ $$NPV = (Specificity * (1-Prevalence)) / (((1-Sensitivity) * Prevalence) + ((Specificity) * (1-Prevalence)))$$ See references discussions statistics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"references","dir":"Reference","previous_headings":"","what":"References","title":"Specificity — spec","text":"Altman, D.G., Bland, J.M. (1994) ``Diagnostic tests 1: sensitivity specificity,'' British Medical Journal, vol 308, 1552.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"Specificity — spec","text":"Max Kuhn","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/spec.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Specificity — spec","text":"","code":"# Two class data(\"two_class_example\") spec(two_class_example, truth, predicted) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 spec binary 0.793 # Multiclass library(dplyr) data(hpc_cv) hpc_cv %>% filter(Resample == \"Fold01\") %>% spec(obs, pred) #> # A tibble: 1 × 3 #> .metric .estimator .estimate #> #> 1 spec macro 0.886 # Groups are respected hpc_cv %>% group_by(Resample) %>% spec(obs, pred) #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 spec macro 0.886 #> 2 Fold02 spec macro 0.882 #> 3 Fold03 spec macro 0.899 #> 4 Fold04 spec macro 0.879 #> 5 Fold05 spec macro 0.881 #> 6 Fold06 spec macro 0.873 #> 7 Fold07 spec macro 0.866 #> 8 Fold08 spec macro 0.884 #> 9 Fold09 spec macro 0.867 #> 10 Fold10 spec macro 0.875 # Weighted macro averaging hpc_cv %>% group_by(Resample) %>% spec(obs, pred, estimator = \"macro_weighted\") #> # A tibble: 10 × 4 #> Resample .metric .estimator .estimate #> #> 1 Fold01 spec macro_weighted 0.816 #> 2 Fold02 spec macro_weighted 0.815 #> 3 Fold03 spec macro_weighted 0.839 #> 4 Fold04 spec macro_weighted 0.803 #> 5 Fold05 spec macro_weighted 0.812 #> 6 Fold06 spec macro_weighted 0.795 #> 7 Fold07 spec macro_weighted 0.790 #> 8 Fold08 spec macro_weighted 0.814 #> 9 Fold09 spec macro_weighted 0.795 #> 10 Fold10 spec macro_weighted 0.801 # Vector version spec_vec( two_class_example$truth, two_class_example$predicted ) #> [1] 0.7933884 # Making Class2 the \"relevant\" level spec_vec( two_class_example$truth, two_class_example$predicted, event_level = \"second\" ) #> [1] 0.879845"},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":null,"dir":"Reference","previous_headings":"","what":"Summary Statistics for Confusion Matrices — summary.conf_mat","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"Various statistical summaries confusion matrices produced returned tibble. include shown help pages sens(), recall(), accuracy(), among others.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"","code":"# S3 method for conf_mat summary( object, prevalence = NULL, beta = 1, estimator = NULL, event_level = yardstick_event_level(), ... )"},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"object object class conf_mat(). prevalence number (0, 1) prevalence (.e. prior) event. left default, data used derive value. beta numeric value used weight precision recall f_meas(). estimator One : \"binary\", \"macro\", \"macro_weighted\", \"micro\" specify type averaging done. \"binary\" relevant two class case. three general methods calculating multiclass metrics. default automatically choose \"binary\" \"macro\" based estimate. event_level single string. Either \"first\" \"second\" specify level truth consider \"event\". argument applicable estimator = \"binary\". default uses internal helper generally defaults \"first\", however, deprecated global option yardstick.event_first set, used instead warning. ... currently used.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"tibble containing various classification metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"relevant-level","dir":"Reference","previous_headings":"","what":"Relevant Level","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"common convention factor level automatically considered \"event\" \"positive\" result computing binary classification metrics. yardstick, default use first level. alter , change argument event_level \"second\" consider last level factor level interest. multiclass extensions involving one-vs-comparisons (macro averaging), option ignored \"one\" level always relevant result.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/summary.conf_mat.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Summary Statistics for Confusion Matrices — summary.conf_mat","text":"","code":"data(\"two_class_example\") cmat <- conf_mat(two_class_example, truth = \"truth\", estimate = \"predicted\") summary(cmat) #> # A tibble: 13 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 sens binary 0.880 #> 4 spec binary 0.793 #> 5 ppv binary 0.819 #> 6 npv binary 0.861 #> 7 mcc binary 0.677 #> 8 j_index binary 0.673 #> 9 bal_accuracy binary 0.837 #> 10 detection_prevalence binary 0.554 #> 11 precision binary 0.819 #> 12 recall binary 0.880 #> 13 f_meas binary 0.849 summary(cmat, prevalence = 0.70) #> # A tibble: 13 × 3 #> .metric .estimator .estimate #> #> 1 accuracy binary 0.838 #> 2 kap binary 0.675 #> 3 sens binary 0.880 #> 4 spec binary 0.793 #> 5 ppv binary 0.909 #> 6 npv binary 0.739 #> 7 mcc binary 0.677 #> 8 j_index binary 0.673 #> 9 bal_accuracy binary 0.837 #> 10 detection_prevalence binary 0.554 #> 11 precision binary 0.819 #> 12 recall binary 0.880 #> 13 f_meas binary 0.849 library(dplyr) library(tidyr) data(\"hpc_cv\") # Compute statistics per resample then summarize all_metrics <- hpc_cv %>% group_by(Resample) %>% conf_mat(obs, pred) %>% mutate(summary_tbl = lapply(conf_mat, summary)) %>% unnest(summary_tbl) all_metrics %>% group_by(.metric) %>% summarise( mean = mean(.estimate, na.rm = TRUE), sd = sd(.estimate, na.rm = TRUE) ) #> # A tibble: 13 × 3 #> .metric mean sd #> #> 1 accuracy 0.709 2.47e- 2 #> 2 bal_accuracy 0.720 1.92e- 2 #> 3 detection_prevalence 0.25 9.25e-18 #> 4 f_meas 0.569 3.46e- 2 #> 5 j_index 0.439 3.85e- 2 #> 6 kap 0.508 4.10e- 2 #> 7 mcc 0.515 4.16e- 2 #> 8 npv 0.896 1.11e- 2 #> 9 ppv 0.633 3.87e- 2 #> 10 precision 0.633 3.87e- 2 #> 11 recall 0.560 3.09e- 2 #> 12 sens 0.560 3.09e- 2 #> 13 spec 0.879 9.67e- 3"},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":null,"dir":"Reference","previous_headings":"","what":"Two Class Predictions — two_class_example","title":"Two Class Predictions — two_class_example","text":"Two Class Predictions","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"value","dir":"Reference","previous_headings":"","what":"Value","title":"Two Class Predictions — two_class_example","text":"two_class_example data frame","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"details","dir":"Reference","previous_headings":"","what":"Details","title":"Two Class Predictions — two_class_example","text":"data test set form model built two classes (\"Class1\" \"Class2\"). columns true predicted classes column probabilities class.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/two_class_example.html","id":"ref-examples","dir":"Reference","previous_headings":"","what":"Examples","title":"Two Class Predictions — two_class_example","text":"","code":"data(two_class_example) str(two_class_example) #> 'data.frame':\t500 obs. of 4 variables: #> $ truth : Factor w/ 2 levels \"Class1\",\"Class2\": 2 1 2 1 2 1 1 1 2 2 ... #> $ Class1 : num 0.00359 0.67862 0.11089 0.73516 0.01624 ... #> $ Class2 : num 0.996 0.321 0.889 0.265 0.984 ... #> $ predicted: Factor w/ 2 levels \"Class1\",\"Class2\": 2 1 2 1 2 1 1 1 2 2 ... # `truth` is a 2 level factor. The first level is `\"Class1\"`, which is the # \"event of interest\" by default in yardstick. See the Relevant Level # section in any classification function (such as `?pr_auc`) to see how # to change this. levels(hpc_cv$obs) #> [1] \"VF\" \"F\" \"M\" \"L\""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick-package.html","id":null,"dir":"Reference","previous_headings":"","what":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","title":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","text":"Tidy tools quantifying well model fits data set confusion matrices, class probability curve summaries, regression metrics (e.g., RMSE).","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick-package.html","id":"author","dir":"Reference","previous_headings":"","what":"Author","title":"yardstick: Tidy Characterizations of Model Performance — yardstick-package","text":"Maintainer: Emil Hvitfeldt emil.hvitfeldt@posit.co (ORCID) Authors: Max Kuhn max@posit.co Davis Vaughan davis@posit.co contributors: Posit Software, PBC [copyright holder, funder]","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":null,"dir":"Reference","previous_headings":"","what":"Developer function for handling missing values in new metrics — yardstick_remove_missing","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"yardstick_remove_missing(), yardstick_any_missing() useful alongside metric-summarizers functions implementing new custom metrics. yardstick_remove_missing() removes observations contains missing values across, truth, estimate case_weights. yardstick_any_missing() returns FALSE missing values inputs.","code":""},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":"ref-usage","dir":"Reference","previous_headings":"","what":"Usage","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"","code":"yardstick_remove_missing(truth, estimate, case_weights) yardstick_any_missing(truth, estimate, case_weights)"},{"path":"https://yardstick.tidymodels.org/dev/reference/yardstick_remove_missing.html","id":"arguments","dir":"Reference","previous_headings":"","what":"Arguments","title":"Developer function for handling missing values in new metrics — yardstick_remove_missing","text":"truth, estimate Vectors length. case_weights vector length truth estimate, NULL case weights used.","code":""},{"path":[]},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-development-version","dir":"Changelog","previous_headings":"","what":"yardstick (development version)","title":"yardstick (development version)","text":"Brier score survival data added brier_survival(). Integrated Brier score survival data added brier_survival_integrated(). Concordance index survival data added concordance_survival(). Time-Dependent ROC curves estimation right-censored data can now calculated roc_curve_survival(). Time-Dependent ROC AUC estimation right-censored data can now calculated roc_auc_survival(). metric_set() can now used combination dynamic static survival metrics. demographic_parity(), equalized_odds(), equal_opportunity() new metrics measuring model fairness. implemented new_groupwise_metric() constructor, general interface defining group-aware metrics allows quickly flexibly defining fairness metrics problem context mind. Added print method metrics metric sets (#435). warnings errors updated use cli package increased clarity consistency. (#456, #457, #458)","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-120","dir":"Changelog","previous_headings":"","what":"yardstick 1.2.0","title":"yardstick 1.2.0","text":"CRAN release: 2023-04-21","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-1-2-0","dir":"Changelog","previous_headings":"","what":"New Metrics","title":"yardstick 1.2.0","text":"Brier score classification added brier_class() (#139).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"improvements-1-2-0","dir":"Changelog","previous_headings":"","what":"Improvements","title":"yardstick 1.2.0","text":"global option, yardstick.event_first, hard deprecated favor using explicit argument, event_level. Setting option now produce warning, won’t effect. (#173) Removed start-message event_level argument. yardstick metric functions now use pure tidyselect interface truth, estimate, ... class probability metrics (#322). Changed default aspect ratio PR curves 1.0. Error messages now show user-facing function called (#348). classification probability metrics now fully support class_pred objects {probably} package (#341).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-1-2-0","dir":"Changelog","previous_headings":"","what":"Bug Fixes","title":"yardstick 1.2.0","text":"Using metric_set() metric created metric_tweak() longer produces error, favor arguments set metric_tweak() (#351). Metric summarizers longer error column names data conflict argument names (#382). conf_mat() longer throw errors listed internal (#327).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"developer-1-2-0","dir":"Changelog","previous_headings":"","what":"Developer","title":"yardstick 1.2.0","text":"metric_vec_template() soft deprecated favor manual flexible metric creation approach. yardstick_remove_missing() yardstick_any_missing() added treatment missing values. check_class_metric(), check_numeric_metric(), check_prob_metric() added perform standardized input checking classification, regression class probability metrics respectively. changes mean developer’s responsibility perform validation truth estimate input. (#337). metric_summarizer() soft deprecated favor specific newly added class_metric_summarizer(), numeric_metric_summarizer(), prob_metric_summarizer(), curve_metric_summarizer() (#322). dots_to_estimate() soft deprecated along metric_summarizer(). dots_to_estimate() needed prob_metric_summarizer(), curve_metric_summarizer() (#329).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-110","dir":"Changelog","previous_headings":"","what":"yardstick 1.1.0","title":"yardstick 1.1.0","text":"CRAN release: 2022-09-07 Emil Hvitfeldt now maintainer (#315). Improved chained error thrown metric_set() one metric computations fails (#313).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-100","dir":"Changelog","previous_headings":"","what":"yardstick 1.0.0","title":"yardstick 1.0.0","text":"CRAN release: 2022-06-06 yardstick metrics now support case weights new case_weights argument. also includes metric-adjacent functions like roc_curve(), pr_curve(), conf_mat(), metric_set(). options argument roc_curve(), roc_auc(), roc_aunp(), roc_aunu(), metrics() passed along pROC package now deprecated longer affect. result changing ROC curve implementation supports case weights, support previous options. need options, suggest wrapping pROC custom metric (#296). conf_mat() now ignores inputs passed ... warns try thing. Previously, passed base::table(), addition case weight support, table() longer used (#295). Fixed small mistake ccc() unbiased covariance wasn’t used bias = FALSE. j_index() now throws correct warning 0 denominator computing sens() internally. Additionally, multiclass case now removes levels occurs multiclass weighted average computation, consistent metrics updated handle #118 (#265). Improved possible ambiguity documentation data argument metrics (#255). purrr removed Suggests. pROC package removed dependency (#300). Moved Custom Metrics vignette tidymodels.org (#236).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-009","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.9","title":"yardstick 0.0.9","text":"CRAN release: 2021-11-22 New metric poisson_log_loss() added (#146). sensitivity() specificity() now work correctly tune workflowsets packages (#232). roc_curve() now throws informative error truth doesn’t control event observations. dplyr 1.0.0 now required. allowed us remove multiple usages dplyr::() favor dplyr::summarise(), can now return packed data frame columns multiple rows per group. Removed internal hardcoding \"dplyr_error\" avoid issues upcoming dplyr 1.0.8 release (#244). Updated test suite testthat 3e (#243). Internal upkeep done move rlang::warn(.subclass = ) rlang::warn(class = ), since .subclass argument deprecated (#225).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-008","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.8","title":"yardstick 0.0.8","text":"CRAN release: 2021-03-28 New metric_tweak() adjusting default values optional arguments existing yardstick metric. useful quickly adjust defaults metric included metric_set(), especially metric set going used tuning tune package (#206, #182). New classification_cost() metric computing cost poor class probability prediction using user-defined costs (#3). New msd() computing mean signed deviation (also called mean signed difference, mean signed error) (#183, @datenzauberai). class_pred objects probably package now supported, automatically converted factors computing metric. Note means equivocal values materialized NA (#198). kap() metric new weighting argument apply linear quadratic weightings computing kappa value (#2, #125, @jonthegeek). sens() undefined computing ppv(), npv(), j_index(), bal_accuracy(), sensitivity warning now correctly thrown, rather recall warning (#101). autoplot() method gain curves now plots curve line top shaded polygon, resulting sharper look line (#192, @eddjberry). autoplot() methods conf_mat now respect user-defined dimension names added conf_mat(dnn = ) converting table dimension names conf_mat (#191). Added as_tibble() method metric_set objects. Printing metric_set now uses print tibble rather data frame (#186). Re-licensed package GPL-2 MIT. See consent copyright holders (#204).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-007","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.7","title":"yardstick 0.0.7","text":"CRAN release: 2020-07-13 global option, yardstick.event_first, deprecated favor new explicit argument, event_level. metric functions previously supported changing “event” level gained new argument. global option historical design decision can classified case hidden argument. Existing code relied global option continue work version yardstick, however now get -per-session warning requests update instead use explicit event_level argument. global option completely removed future version. update, follow guide (#163). roc_auc() Hand-Till multiclass estimator now ignore levels truth occur zero times actual data. methods multiclass averaging, usually returns NA, however, ignoring levels manner consistent implementations HandTill2001 pROC packages (#123). roc_auc() roc_curve() now set direction = \"<\" computing ROC curve using pROC::roc(). Results computed incorrectly direction = \"auto\" probability values predicting wrong class (#123). mn_log_loss() now respects (deprecated) global option yardstick.event_first. However, instead change relevant event level event_level argument. metric_set() now strips package name auto-labeling functions (@rorynolan, #151). three new helper functions easily creating custom metric functions: new_class_metric(), new_prob_metric(), new_numeric_metric(). Rcpp removed direct dependency.","code":"`options(yardstick.event_first = TRUE)` -> `event_level = \"first\"` (the default) `options(yardstick.event_first = FALSE)` -> `event_level = \"second\"`"},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-006","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.6","title":"yardstick 0.0.6","text":"CRAN release: 2020-03-17 roc_auc() now warns events controls provided truth column, returns NA (@dpastling, #132). Adds sensitivity() specificity() aliases sens() spec() respectively, avoids conflict packages e.g. readr::spec(). roc_aunu() roc_aunp() two new ROC AUC metrics multiclass classifiers. measure AUC class rest, roc_aunu() using uniform class distribution (#69) roc_aunp() using priori class distribution (#70).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-005","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.5","title":"yardstick 0.0.5","text":"CRAN release: 2020-01-23","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-5","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.5","text":"autoplot() heat map confusion matrices now places predicted values x axis truth values y axis consistent confusion matrix print() method. autoplot() mosaic plot confusion matrices x y axis labels backwards. corrected.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-004","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.4","title":"yardstick 0.0.4","text":"CRAN release: 2019-08-26","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-4","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.4","text":"iic() new numeric metric computing index ideality correlation. can seen potential alternative traditional correlation coefficient, used QSAR models (@jyuu, #115). average_precision() new probability metric can used alternative pr_auc(). benefit avoiding issues ambiguity case recall == 0 current number false positives 0.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-4","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.4","text":"metric_set() output now includes metrics attribute contains list original metric functions used generate metric set. metric function now direction attribute attached , specifying whether minimize maximize metric. Classification metrics can potentially 0 value denominator now throw informative warning case occurs. include recall(), precision(), sens(), spec() (#98). autoplot() method pr_curve() improved always set axis limits c(0, 1). valid arguments pROC::roc() now utilized, including passed pROC::auc(). Documentation class probability metrics improved informative examples (@rudeboybert, #100).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-0-0-4","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"yardstick 0.0.4","text":"mn_log_loss() now uses min/max rule computing log estimated probabilities avoid problematic undefined log values (#103). pr_curve() now places 1 first precision value, rather NA. NA technically correct precision undefined , 1 practically correct generates correct PR Curve graph , importantly, allows pr_auc() compute correct AUC. pr_curve() generate wrong results somewhat rare case two class probability estimates , different truth values. pr_curve() (subsequently pr_auc()) now generates correct curve duplicate class probability values (reported @dariyasydykova, #93). Binary mcc() now avoids integer overflow confusion matrix elements large (#108).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-003","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.3","title":"yardstick 0.0.3","text":"CRAN release: 2019-03-08","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-3","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.3","text":"mase() numeric metric mean absolute scaled error. generally useful forecasting time series (@alexhallam, #68). huber_loss() numeric metric less sensitive outliers rmse(), sensitive mae() small errors (@blairj09, #71). huber_loss_pseudo() smoothed form huber_loss() (@blairj09, #71). smape() numeric metric based percentage errors (@riazhedayati, #67). conf_mat objects now two ggplot2::autoplot() methods easy visualization confusion matrix either heat map mosaic plot (@EmilHvitfeldt, #10).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-3","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.3","text":"metric_set() now returns classed function. numeric metrics used, \"numeric_metric_set\" function returned. class probability metrics used, \"class_prob_metric_set\" returned.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"bug-fixes-0-0-3","dir":"Changelog","previous_headings":"","what":"Bug fixes","title":"yardstick 0.0.3","text":"Tests related fixed R 3.6 sample() function fixed. f_meas() propagates NA values precision() recall() correctly (#77). \"micro\" estimators now propagate NA values correctly. roc_auc(estimator = \"hand_till\") now correctly computes metric column names probability matrix exact levels truth. Note computation still assumes order supplied probability matrix columns still matches order levels(truth), like multiclass metrics (#86).","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-002","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.2","title":"yardstick 0.0.2","text":"CRAN release: 2018-11-05","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"breaking-changes-0-0-2","dir":"Changelog","previous_headings":"","what":"Breaking changes","title":"yardstick 0.0.2","text":"desire standardize yardstick API drove breaking changes. output metric now line tidy principles, returning tibble rather single numeric. Additionally, metrics now standard argument list able switch metrics combine together effortlessly. metrics now return tibble rather single numeric value. format allows metrics work grouped data frames (resamples). also allows bundle multiple metrics together new function, metric_set(). class probability metrics, now 1 column can passed ... binary implementation used. metrics longer select first column multiple columns supplied, instead throw error. summary() method conf_mat objects now returns tibble consistent change metric functions. naming consistency, mnLogLoss() renamed mn_log_loss() mn_log_loss() now returns negative log loss multinomial distribution. argument na.rm changed na_rm metrics align tidymodels model implementation principles.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"core-features-0-0-2","dir":"Changelog","previous_headings":"","what":"Core features","title":"yardstick 0.0.2","text":"metric now vector interface go alongside data frame interface. vector functions end _vec(). vector interface accepts vector/matrix inputs returns single numeric value. Multiclass support added classification metric. support varies one metric next, generally macro micro averaging available metrics, metrics specialized multiclass implementations (example, roc_auc() supports multiclass generalization presented paper Hand Till). information, see vignette(\"multiclass\", \"yardstick\"). metrics now work grouped data frames. produces tibble many rows groups, useful used alongside resampling techniques.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"new-metrics-and-functionality-0-0-2","dir":"Changelog","previous_headings":"","what":"New metrics and functionality","title":"yardstick 0.0.2","text":"mape() calculates mean absolute percent error. kap() metric similar accuracy() calculates Cohen’s kappa. detection_prevalence() calculates number predicted positive events relative total number predictions. bal_accuracy() calculates balanced accuracy average sensitivity specificity. roc_curve() calculates receiver operator curves returns results tibble. pr_curve() calculates precision recall curves. gain_curve() lift_curve() calculate information used gain lift curves. gain_capture() measure performance similar spirit AUC applied gain curve. pr_curve(), roc_curve(), gain_curve() lift_curve() ggplot2::autoplot() methods easy visualization. metric_set() constructs functions calculate multiple metrics .","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"other-improvements-0-0-2","dir":"Changelog","previous_headings":"","what":"Other improvements","title":"yardstick 0.0.2","text":"infrastructure creating metrics exposed allow users extend yardstick work metrics. might want want metrics work grouped data frames box, want standardization error checking yardstick already provides. See vignette(\"custom-metrics\", \"yardstick\") examples. vignette describing three classes metrics used yardstick added. also includes list every metric available, grouped class. See vignette(\"metric-types\", \"yardstick\"). error messages yardstick now much informative, better feedback types input metric can use kinds metrics can used together (.e. metric_set()). now grouped_df method conf_mat() returns tibble list column conf_mat objects. metric now help page. allows us better document nuances metric without cluttering help pages metrics.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"dependencies-0-0-2","dir":"Changelog","previous_headings":"","what":"Dependencies","title":"yardstick 0.0.2","text":"broom removed Depends, replaced generics Suggests. tidyr ggplot2 moved Suggests. MLmetrics removed dependency.","code":""},{"path":"https://yardstick.tidymodels.org/dev/news/index.html","id":"yardstick-001","dir":"Changelog","previous_headings":"","what":"yardstick 0.0.1","title":"yardstick 0.0.1","text":"CRAN release: 2017-11-12 First CRAN release","code":""}]