text:
2013.01.017},
  author = {O'Brien, Keith and Rani, Sweta and Corcoran, Claire and Wallace, Robert and Hughes, Linda and Friel, Anne M. and McDonnell, Susan and Crown, John and Radomski, Marek W. and O'Driscoll, Lorraine},
  month = may,
  year = {2013},
  pages = {1845-1859},
  file = {C:\\Users\\charl\\OneDrive\\zotero\\O’Brien et al_2013_Exosomes from triple-negative breast.pdf}
}

@article{li2009,
  title = {Claudin-Containing Exosomes in the Peripheral Circulation of Women with Ovarian Cancer},
  volume = {9},
  issn = {1471-2407},
  abstract = {The absence of highly sensitive and specific serum biomarkers makes mass screening for ovarian cancer impossible. The claudin proteins are frequently overexpressed in ovarian cancers, but their potential as prognostic, diagnostic, or detection markers remains unclear. Here, we have explored the possible use of these proteins as screening biomarkers for ovarian cancer detection.},
  number = {1},
  journal = {BMC Cancer},
  doi = {10.1186/1471-2407-9-244},
  author = {Li, Jianghong and {Sherman-Baust}, Cheryl A. and {Tsai-Turton}, Miyun and Bristow, Robert E. and Roden, Richard B. and Morin, Patrice J.},
  month = jul,
  year = {2009},
  pages = {244},
  file = {C:\\Users\\charl\\OneDrive\\zotero\\Li et al_2009_Claudin-containing exosomes in the.pdf}
}

@article{nguyen2019,
  title = {Quantifying Murine Placental Extracellular Vesicles across Gestation and in Preterm Birth Data with {{tidyNano}}: {{A}} Computational Framework for Analyzing and Visualizing Nanoparticle Data in {{R}}},
  volume = {14},
  issn = {1932-6203},
  shorttitle = {Quantifying Murine Placental Extracellular Vesicles across Gestation and in Preterm Birth Data with {{tidyNano}}},
  abstract = {Extracellular vesicles (EVs) are increasingly recognized as important mediators of intercellular communication that carry protein, lipids, and nucleic acids via the circulation to target cells whereupon they mediate physiological changes. In pregnancy, EVs are released in high quantities from the placenta and have been postulated to target multiple cell types, including those of the vascular and immune systems. However, most studies of pregnancy-associated EVs have used clinical samples and in vitro models; to date, few studies have taken advantage of murine models in which pregnancy can be precisely timed and manipulated. In this study, we used a murine model to determine whether the quantity of EVs is altered during healthy pregnancy and during inflammation-associated preterm birth. To facilitate data analysis, we developed a novel software package, tidyNano, an R package that provides functions to import, clean, and quickly summarize raw data generated by the nanoparticle tracking device, NanoSight (Malvern Panalytical). We also developed shinySIGHT, a Shiny web application that allows for interactive exploration and visualization of EV data. In mice, EV concentration in blood increased linearly across pregnancy, with significant rises at GD14.5 and 17.5 relative to EV concentrations in nonpregnant females. Additionally, lipopolysaccharide treatment resulted in a significant reduction in circulating EV concentrations relative to vehicle-treated controls at GD16.5 within 4 hours. Use of tidyNano facilitated rapid analysis of EV data; importantly, this package provides a straightforward framework by which diverse types of large datasets can be simply and efficiently analyzed, is freely available under the MIT license, and is hosted on GitHub (https://nguyens7.github.io/tidyNano/). Our data highlight the utility of the mouse as a model of EV biology in pregnancy, and suggest that placental dysfunction is associated with reduced circulating EVs.},
  language = {en},
  number = {6},
  journal = {PLOS ONE},
  doi = {10.1371/journal.pone.0218270},
  author = {Nguyen, Sean L. and Greenberg, Jacob W. and Wang, Hao and Collaer, Benjamin W. and Wang, Jianrong and Petroff, Margaret G.},
  month = jun,
  year = {2019},
  pages = {e0218270},
  file = {C:\\Users\\charl\\OneDrive\\zotero\\Nguyen et al_2019_Quantifying murine placental.pdf}
}

@article{perez-riverol2019,
  title = {The {{PRIDE}} Database and Related Tools and Resources in 2019: Improving Support for Quantification Data},
  volume = {47},
  issn = {0305-1048},
  shorttitle = {The {{PRIDE}} Database and Related Tools and Resources in 2019},
  abstract = {Abstract.  The PRoteomics IDEntifications (PRIDE) database (https://www.ebi.ac.uk/pride/) is the world's largest data repository of mass spectrometry-based prot},
  language = {en},
  number = {D1},
  journal = {Nucleic Acids Research},
  doi = {10.1093/nar/gky1106},
  author = {{Perez-Riverol}, Yasset and Csordas, Attila and Bai, Jingwen and {Bernal-Llinares}, Manuel and Hewapathirana, Suresh and Kundu, Deepti J. and Inuganti, Avinash and Griss, Johannes and Mayer, Gerhard and Eisenacher, Martin and P{\'e}rez, Enrique and Uszkoreit, Julian and Pfeuffer, Julianus and Sachsenberg, Timo and Y\i{}lmaz, {\c S}ule and Tiwary, Shivani and Cox, J{\"u}rgen and Audain, Enrique and Walzer, Mathias and Jarnuczak, Andrew F. and Ternent, Tobias and Brazma, Alvis and Vizca{\'i}no, Juan Antonio},
  month = jan,
  year = {2019},
  pages = {D442-D450},
  file = {C:\\Users\\charl\\OneDrive\\zotero\\Perez-Riverol et al_2019_The PRIDE database and related tools.pdf}
}

@article{deutsch2017,
  title = {The {{ProteomeXchange}} Consortium in 2017: Supporting the Cultural Change in Proteomics Public Data Deposition},
  volume = {45},
  issn = {0305-1048},
  shorttitle = {The {{ProteomeXchange}} Consortium in 2017},
  abstract = {Abstract.  The ProteomeXchange (PX) Consortium of proteomics resources (http://www.proteomexchange.org) was formally started in 2011 to standardize data submiss},
  language = {en},
  number = {D1},
  journal = {Nucleic Acids Research},
  doi = {10.1093/nar/gkw936},
  author = {Deutsch, Eric W. and Csordas, Attila and Sun, Zhi and Jarnuczak, Andrew and {Perez-Riverol}, Yasset and Ternent, Tobias and Campbell, David S. and {Bernal-Llinares}, Manuel and Okuda, Shujiro and Kawano, Shin and Moritz, Robert L. and Carver, Jeremy J. and Wang, Mingxun and Ishihama, Yasushi and Bandeira, Nuno and Hermjakob, Henning and Vizca{\'i}no, Juan Antonio},
  month = jan,
  year = {2017},
  pages = {D1100-D1106},
  file = {C:\\Users\\charl\\OneDrive\\zotero\\Deutsch et al_2017_The ProteomeXchange consortium in 2017.pdf}
}

repository: csdaw/candidaev

repo_name: csdaw/candidaev


text:
@article{Wang_2011_apr,
 author = {Xiangfeng Wang and John D. Laurie and Tao Liu and Jacqueline Wentz and X. Shirley Liu},
 doi = {10.1016/j.ygeno.2011.01.006},
 journal = {Genomics},
 month = {apr},
 number = {4},
 pages = {235--243},
 publisher = {Elsevier BV},
 title = {Computational dissection of Arabidopsis smRNAome leads to discovery of novel microRNAs and short interfering RNAs associated with transcription start sites},
 url = {https://doi.org/10.1016%2Fj.ygeno.2011.01.006},
 volume = {97},
 year = {2011}
}

license: MIT

repository: wanglab-cau/CrazyHsu

license_type: permissive

repo_name: wanglab-cau/CrazyHsu

branch_name: refs/heads/main

extension: bib


text:
% Encoding: UTF-8

@Article{Dabney2020,
  author    = {Will Dabney and Zeb Kurth-Nelson and Naoshige Uchida and Clara Kwon Starkweather and Demis Hassabis and R{\'{e}}mi Munos and Matthew Botvinick},
  journal   = {Nature},
  title     = {A distributional code for value in dopamine-based reinforcement learning},
  year      = {2020},
  month     = jan,
  number    = {7792},
  pages     = {671--675},
  volume    = {577},
  doi       = {10.1038/s41586-019-1924-6},
  publisher = {Springer Science and Business Media {LLC}},
}

@Misc{fujimoto2018addressing,
  author        = {Scott Fujimoto and Herke van Hoof and David Meger},
  title         = {{Addressing function approximation error in actor-critic methods}},
  year          = {2018},
  archiveprefix = {arXiv},
  eprint        = {1802.09477},
  primaryclass  = {cs.AI},
}

@Misc{Hirlaender2013aa,
  author       = {Hirlaender, S.},
  howpublished = {\url{https://github.com/MathPhysSim/PER-NAF}},
  title        = {{PER-NAF}},
  year         = {2013},
  commit       = {4f57d6a0e4c030202a07a60bc1bb1ed1544bf679},
  journal      = {GitHub repository},
  publisher    = {GitHub},
}

@Article{Kumar2020,
  author      = {Aviral Kumar and Abhishek Gupta and Sergey Levine},
  journal     = {{}},
  title       = {{DisCor: corrective feedback in reinforcement learning via distribution correction}},
  abstract    = {Deep reinforcement learning can learn effective policies for a wide range of tasks, but is notoriously difficult to use due to instability and sensitivity to hyperparameters. The reasons for this remain unclear. When using standard supervised methods (e.g., for bandits), on-policy data collection provides "hard negatives" that correct the model in precisely those states and actions that the policy is likely to visit. We call this phenomenon "corrective feedback." We show that bootstrapping-based Q-learning algorithms do not necessarily benefit from this corrective feedback, and training on the experience collected by the algorithm is not sufficient to correct errors in the Q-function. In fact, Q-learning and related methods can exhibit pathological interactions between the distribution of experience collected by the agent and the policy induced by training on that experience, leading to potential instability, sub-optimal convergence, and poor results when learning from noisy, sparse or delayed rewards. We demonstrate the existence of this problem, both theoretically and empirically. We then show that a specific correction to the data distribution can mitigate this issue. Based on these observations, we propose a new algorithm, DisCor, which computes an approximation to this optimal distribution and uses it to re-weight the transitions used for training, resulting in substantial improvements in a range of challenging RL settings, such as multi-task learning and learning from noisy reward signals. Blog post presenting a summary of this work is available at: https://bair.berkeley.edu/blog/2020/03/16/discor/.},
  date        = {2020-03-16},
  eprint      = {2003.07305v1},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2003.07305v1:PDF},
  keywords    = {cs.LG, stat.ML},
}

@Article{Kumar2019,
  author      = {Aviral Kumar and Justin Fu and George Tucker and Sergey Levine},
  journal     = {{}},
  title       = {Stabilizing Off-Policy Q-Learning via Bootstrapping Error Reduction},
  abstract    = {Off-policy reinforcement learning aims to leverage experience collected from prior policies for sample-efficient learning. However, in practice, commonly used off-policy approximate dynamic programming methods based on Q-learning and actor-critic methods are highly sensitive to the data distribution, and can make only limited progress without collecting additional on-policy data. As a step towards more robust off-policy algorithms, we study the setting where the off-policy experience is fixed and there is no further interaction with the environment. We identify bootstrapping error as a key source of instability in current methods. Bootstrapping error is due to bootstrapping from actions that lie outside of the training data distribution, and it accumulates via the Bellman backup operator. We theoretically analyze bootstrapping error, and demonstrate how carefully constraining action selection in the backup can mitigate it. Based on our analysis, we propose a practical algorithm, bootstrapping error accumulation reduction (BEAR). We demonstrate that BEAR is able to learn robustly from different off-policy distributions, including random and suboptimal demonstrations, on a range of continuous control tasks.},
  date        = {2019-06-03},
  eprint      = {1906.00949v2},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1906.00949v2:PDF},
  keywords    = {cs.LG, stat.ML},
}

@Article{Schaul2015,
  author      = {Tom Schaul and John Quan and Ioannis Antonoglou and David Silver},
  journal     = {{}},
  title       = {Prioritized Experience Replay},
  abstract    = {Experience replay lets online reinforcement learning agents remember and reuse experiences from the past. In prior work, experience transitions were uniformly sampled from a replay memory. However, this approach simply replays transitions at the same frequency that they were originally experienced, regardless of their significance. In this paper we develop a framework for prioritizing experience, so as to replay important transitions more frequently, and therefore learn more efficiently. We use prioritized experience replay in Deep Q-Networks (DQN), a reinforcement learning algorithm that achieved human-level performance across many Atari games. DQN with prioritized experience replay achieves a new state-of-the-art, outperforming DQN with uniform replay on 41 out of 49 games.},
  date        = {2015-11-18},
  eprint      = {1511.05952v4},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1511.05952v4:PDF},
  keywords    = {cs.LG},
}

@Article{Schulman2017,
  author        = {John Schulman and Filip Wolski and Prafulla Dhariwal and Alec Radford and Oleg Klimov},
  journal       = {CoRR},
  title         = {Proximal Policy Optimization Algorithms},
  year          = {2017},
  volume        = {abs/1707.06347},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/SchulmanWDRK17.bib},
  eprint        = {1707.06347},
  timestamp     = {Mon, 13 Aug 2018 16:47:34 +0200},
  url           = {http://arxiv.org/abs/1707.06347},
}

@Article{Wang2019,
  author      = {Tingwu Wang and Xuchan Bao and Ignasi Clavera and Jerrick Hoang and Yeming Wen and Eric Langlois and Shunshi Zhang and Guodong Zhang and Pieter Abbeel and Jimmy Ba},
  journal     = {{}},
  title       = {Benchmarking Model-Based Reinforcement Learning},
  year        = {2019},
  abstract    = {Model-based reinforcement learning (MBRL) is widely seen as having the potential to be significantly more sample efficient than model-free RL. However, research in model-based RL has not been very standardized. It is fairly common for authors to experiment with self-designed environments, and there are several separate lines of research, which are sometimes closed-sourced or not reproducible. Accordingly, it is an open question how these various existing MBRL algorithms perform relative to each other. To facilitate research in MBRL, in this paper we gather a wide collection of MBRL algorithms and propose over 18 benchmarking environments specially designed for MBRL. We benchmark these algorithms with unified problem settings, including noisy environments. Beyond cataloguing performance, we explore and unify the underlying algorithmic differences across MBRL algorithms. We characterize three key research challenges for future MBRL research: the dynamics bottleneck, the planning horizon dilemma, and the early-termination dilemma. Finally, to maximally facilitate future research on MBRL, we open-source our benchmark in http://www.cs.toronto.edu/~tingwuwang/mbrl.html.},
  date        = {2019-07-03},
  eprint      = {1907.02057v1},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1907.02057v1:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@Article{Pearce2018,
  author       = {Tim Pearce and Felix Leibfried and Alexandra Brintrup and Mohamed Zaki and Andy Neely},
  journal      = {{}},
  title        = {Uncertainty in neural networks: approximately bayesian ensembling},
  year         = {2018},
  abstract     = {Understanding the uncertainty of a neural network's (NN) predictions is essential for many purposes. The Bayesian framework provides a principled approach to this, however applying it to NNs is challenging due to large numbers of parameters and data. Ensembling NNs provides an easily implementable, scalable method for uncertainty quantification, however, it has been criticised for not being Bayesian. This work proposes one modification to the usual process that we argue does result in approximate Bayesian inference; regularising parameters about values drawn from a distribution which can be set equal to the prior. A theoretical analysis of the procedure in a simplified setting suggests the recovered posterior is centred correctly but tends to have an underestimated marginal variance, and overestimated correlation. However, two conditions can lead to exact recovery. We argue that these conditions are partially present in NNs. Empirical evaluations demonstrate it has an advantage over standard ensembling, and is competitive with variational methods.},
  date         = {2018-10-12},
  eprint       = {1810.05546},
  eprintclass  = {stat.ML},
  eprinttype   = {arXiv},
  file         = {:http\://arxiv.org/pdf/1810.05546v5:PDF},
  journaltitle = {The 23rd International Conference on Artificial Intelligence and Statistics, AISTATS 2020},
  keywords     = {stat.ML, cs.LG},
}

@Article{Levine2020,
  author      = {Sergey Levine and Aviral Kumar and George Tucker and Justin Fu},
  journal     = {{}},
  title       = {{Offline reinforcement learning: tutorial, review, and perspectives on open problems}},
  year        = {2020},
  abstract    = {In this tutorial article, we aim to provide the reader with the conceptual tools needed to get started on research on offline reinforcement learning algorithms: reinforcement learning algorithms that utilize previously collected data, without additional online data collection. Offline reinforcement learning algorithms hold tremendous promise for making it possible to turn large datasets into powerful decision making engines. Effective offline reinforcement learning methods would be able to extract policies with the maximum possible utility out of the available data, thereby allowing automation of a wide range of decision-making domains, from healthcare and education to robotics. However, the limitations of current algorithms make this difficult. We will aim to provide the reader with an understanding of these challenges, particularly in the context of modern deep reinforcement learning methods, and describe some potential solutions that have been explored in recent work to mitigate these challenges, along with recent applications, and a discussion of perspectives on open problems in the field.},
  date        = {2020-05-04},
  eprint      = {2005.01643},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2005.01643v3:PDF;:http\://arxiv.org/pdf/2005.01643v3:},
  keywords    = {cs.LG, cs.AI, stat.ML},
}

@Article{Kurutach2018,
  author      = {Thanard Kurutach and Ignasi Clavera and Yan Duan and Aviv Tamar and Pieter Abbeel},
  journal     = {{}},
  title       = {{Model-ensemble trust-region policy optimization}},
  year        = {2018},
  abstract    = {Model-free reinforcement learning (RL) methods are succeeding in a growing number of tasks, aided by recent advances in deep learning. However, they tend to suffer from high sample complexity, which hinders their use in real-world domains. Alternatively, model-based reinforcement learning promises to reduce sample complexity, but tends to require careful tuning and to date have succeeded mainly in restrictive domains where simple models are sufficient for learning. In this paper, we analyze the behavior of vanilla model-based reinforcement learning methods when deep neural networks are used to learn both the model and the policy, and show that the learned policy tends to exploit regions where insufficient data is available for the model to be learned, causing instability in training. To overcome this issue, we propose to use an ensemble of models to maintain the model uncertainty and regularize the learning process. We further show that the use of likelihood ratio derivatives yields much more stable learning than backpropagation through time. Altogether, our approach Model-Ensemble Trust-Region Policy Optimization (ME-TRPO) significantly reduces the sample complexity compared to model-free deep RL methods on challenging continuous control benchmark tasks.},
  date        = {2018-02-28},
  eprint      = {1802.10592},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1802.10592v2:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO},
}

@Article{Haarnoja2018a,
  author      = {Tuomas Haarnoja and Aurick Zhou and Kristian Hartikainen and George Tucker and Sehoon Ha and Jie Tan and Vikash Kumar and Henry Zhu and Abhishek Gupta and Pieter Abbeel and Sergey Levine},
  journal     = {{}},
  title       = {Soft Actor-Critic Algorithms and Applications},
  year        = {2018},
  abstract    = {Model-free deep reinforcement learning (RL) algorithms have been successfully applied to a range of challenging sequential decision making and control tasks. However, these methods typically suffer from two major challenges: high sample complexity and brittleness to hyperparameters. Both of these challenges limit the applicability of such methods to real-world domains. In this paper, we describe Soft Actor-Critic (SAC), our recently introduced off-policy actor-critic algorithm based on the maximum entropy RL framework. In this framework, the actor aims to simultaneously maximize expected return and entropy. That is, to succeed at the task while acting as randomly as possible. We extend SAC to incorporate a number of modifications that accelerate training and improve stability with respect to the hyperparameters, including a constrained formulation that automatically tunes the temperature hyperparameter. We systematically evaluate SAC on a range of benchmark tasks, as well as real-world challenging tasks such as locomotion for a quadrupedal robot and robotic manipulation with a dexterous hand. With these improvements, SAC achieves state-of-the-art performance, outperforming prior on-policy and off-policy methods in sample-efficiency and asymptotic performance. Furthermore, we demonstrate that, in contrast to other off-policy algorithms, our approach is very stable, achieving similar performance across different random seeds. These results suggest that SAC is a promising candidate for learning in real-world robotics tasks.},
  date        = {2018-12-13},
  eprint      = {1812.05905},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1812.05905v2:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@Misc{stable-baselines,
  author       = {Hill, Ashley and Raffin, Antonin and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Traore, Rene and Dhariwal, Prafulla and Hesse, Christopher and Klimov, Oleg and Nichol, Alex and Plappert, Matthias and Radford, Alec and Schulman, John and Sidor, Szymon and Wu, Yuhuai},
  howpublished = {\url{https://github.com/hill-a/stable-baselines}},
  title        = {Stable Baselines},
  year         = {2018},
  journal      = {GitHub repository},
  publisher    = {GitHub},
}

@Article{Schulman2015,
  author      = {John Schulman and Sergey Levine and Philipp Moritz and Michael I. Jordan and Pieter Abbeel},
  journal     = {{}},
  title       = {Trust Region Policy Optimization},
  year        = {2015},
  abstract    = {We describe an iterative procedure for optimizing policies, with guaranteed monotonic improvement. By making several approximations to the theoretically-justified procedure, we develop a practical algorithm, called Trust Region Policy Optimization (TRPO). This algorithm is similar to natural policy gradient methods and is effective for optimizing large nonlinear policies such as neural networks. Our experiments demonstrate its robust performance on a wide variety of tasks: learning simulated robotic swimming, hopping, and walking gaits; and playing Atari games using images of the screen as input. Despite its approximations that deviate from the theory, TRPO tends to give monotonic improvement, with little tuning of hyperparameters.},
  date        = {2015-02-19},
  eprint      = {1502.05477},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1502.05477v5:PDF},
  keywords    = {cs.LG},
}

@Article{Sutton1991,
  author    = {Richard S. Sutton},
  journal   = {{ACM} {SIGART} Bulletin},
  title     = {Dyna, an integrated architecture for learning, planning, and reacting},
  year      = {1991},
  month     = {jul},
  number    = {4},
  pages     = {160--163},
  volume    = {2},
  doi       = {10.1145/122344.122377},
  publisher = {Association for Computing Machinery ({ACM})},
}
@inproceedings{NIPS2010_091d584f,
 author = {Hasselt, Hado},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {J. Lafferty and C. Williams and J. Shawe-Taylor and R. Zemel and A. Culotta},
 pages = {2613--2621},
 publisher = {Curran Associates, Inc.},
 title = {Double Q-learning},
 url = {https://proceedings.neurips.cc/paper/2010/file/091d584fced301b442654dd8c23b3fc9-Paper.pdf},
 volume = {23},
 year = {2010}
}

@Article{Pearce2018a,
  author      = {Tim Pearce and Nicolas Anastassacos and Mohamed Zaki and Andy Neely},
  journal     = {{}},
  title       = {Bayesian Inference with Anchored Ensembles of Neural Networks, and Application to Exploration in Reinforcement Learning},
  abstract    = {The use of ensembles of neural networks (NNs) for the quantification of predictive uncertainty is widespread. However, the current justification is intuitive rather than analytical. This work proposes one minor modification to the normal ensembling methodology, which we prove allows the ensemble to perform Bayesian inference, hence converging to the corresponding Gaussian Process as both the total number of NNs, and the size of each, tend to infinity. This working paper provides early-stage results in a reinforcement learning setting, analysing the practicality of the technique for an ensemble of small, finite number. Using the uncertainty estimates produced by anchored ensembles to govern the exploration-exploitation process results in steadier, more stable learning.},
  date        = {2018-05-29},
  eprint      = {1805.11324},
  eprintclass = {stat.ML},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1805.11324v3:PDF},
  keywords    = {stat.ML, cs.AI, cs.LG},
}

@InProceedings{Furutaa,
  author    = {K. Furuta and M. Yamakita and S. Kobayashi},
  booktitle = {Proceedings {IECON} {\textquotesingle}91: 1991 International Conference on Industrial Electronics, Control and Instrumentation},
  title     = {Swing up control of inverted pendulum},
  year      = {1991},
  publisher = {{IEEE}},
  doi       = {10.1109/iecon.1991.239008},
}

@InProceedings{Silver2014,
  author    = {Silver, David and Lever, Guy and Heess, Nicolas and Degris, Thomas and Wierstra, Daan and Riedmiller, Martin},
  booktitle = {Proceedings of the 31st International Conference on International Conference on Machine Learning - Volume 32},
  title     = {{Deterministic policy gradient algorithms}},
  year      = {2014},
  pages     = {I–387–I–395},
  publisher = {JMLR.org},
  series    = {ICML14},
  abstract  = {In this paper we consider deterministic policy gradient algorithms for reinforcement learning with continuous actions. The deterministic policy gradient has a particularly appealing form: it is the expected gradient of the action-value function. This simple form means that the deterministic policy gradient can be estimated much more efficiently than the usual stochastic policy gradient. To ensure adequate exploration, we introduce an off-policy actor-critic algorithm that learns a deterministic target policy from an exploratory behaviour policy. We demonstrate that deterministic policy gradient algorithms can significantly outperform their stochastic counterparts in high-dimensional action spaces.},
  location  = {Beijing, China},
  numpages  = {9},
}

@Article{BarthMaron2018,
  author      = {Gabriel Barth-Maron and Matthew W. Hoffman and David Budden and Will Dabney and Dan Horgan and Dhruva TB and Alistair Muldal and Nicolas Heess and Timothy Lillicrap},
  journal     = {{}},
  title       = {Distributed Distributional Deterministic Policy Gradients},
  year        = {2018},
  abstract    = {This work adopts the very successful distributional perspective on reinforcement learning and adapts it to the continuous control setting. We combine this within a distributed framework for off-policy learning in order to develop what we call the Distributed Distributional Deep Deterministic Policy Gradient algorithm, D4PG. We also combine this technique with a number of additional, simple improvements such as the use of $N$-step returns and prioritized experience replay. Experimentally we examine the contribution of each of these individual components, and show how they interact, as well as their combined contributions. Our results show that across a wide variety of simple control tasks, difficult manipulation tasks, and a set of hard obstacle-based locomotion tasks the D4PG algorithm achieves state of the art performance.},
  date        = {2018-04-23},
  eprint      = {1804.08617},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1804.08617v1:PDF},
  keywords    = {cs.LG, cs.AI, stat.ML},
}

@Article{Brockman2016,
  author      = {Greg Brockman and Vicki Cheung and Ludwig Pettersson and Jonas Schneider and John Schulman and Jie Tang and Wojciech Zaremba},
  journal     = {{}},
  title       = {OpenAI Gym},
  year        = {2016},
  abstract    = {OpenAI Gym is a toolkit for reinforcement learning research. It includes a growing collection of benchmark problems that expose a common interface, and a website where people can share their results and compare the performance of algorithms. This whitepaper discusses the components of OpenAI Gym and the design decisions that went into the software.},
  date        = {2016-06-05},
  eprint      = {1606.01540},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1606.01540v1:PDF},
  keywords    = {cs.LG, cs.AI},
  url         = {https://github.com/openai/gym/wiki/Leaderboard#pendulum-v0},
}

@Article{Clavera2018,
  author      = {Ignasi Clavera and Jonas Rothfuss and John Schulman and Yasuhiro Fujita and Tamim Asfour and Pieter Abbeel},
  journal     = {{}},
  title       = {Model-Based Reinforcement Learning via Meta-Policy Optimization},
  abstract    = {Model-based reinforcement learning approaches carry the promise of being data efficient. However, due to challenges in learning dynamics models that sufficiently match the real-world dynamics, they struggle to achieve the same asymptotic performance as model-free methods. We propose Model-Based Meta-Policy-Optimization (MB-MPO), an approach that foregoes the strong reliance on accurate learned dynamics models. Using an ensemble of learned dynamic models, MB-MPO meta-learns a policy that can quickly adapt to any model in the ensemble with one policy gradient step. This steers the meta-policy towards internalizing consistent dynamics predictions among the ensemble while shifting the burden of behaving optimally w.r.t. the model discrepancies towards the adaptation step. Our experiments show that MB-MPO is more robust to model imperfections than previous model-based approaches. Finally, we demonstrate that our approach is able to match the asymptotic performance of model-free methods while requiring significantly less experience.},
  date        = {2018-09-14},
  eprint      = {1809.05214},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1809.05214v1:PDF},
  keywords    = {cs.LG, cs.AI, stat.ML},
}

@Article{Zeng2019,
  author      = {Andy Zeng and Shuran Song and Johnny Lee and Alberto Rodriguez and Thomas Funkhouser},
  journal     = {{}},
  title       = {TossingBot: Learning to Throw Arbitrary Objects with Residual Physics},
  abstract    = {We investigate whether a robot arm can learn to pick and throw arbitrary objects into selected boxes quickly and accurately. Throwing has the potential to increase the physical reachability and picking speed of a robot arm. However, precisely throwing arbitrary objects in unstructured settings presents many challenges: from acquiring reliable pre-throw conditions (e.g. initial pose of object in manipulator) to handling varying object-centric properties (e.g. mass distribution, friction, shape) and dynamics (e.g. aerodynamics). In this work, we propose an end-to-end formulation that jointly learns to infer control parameters for grasping and throwing motion primitives from visual observations (images of arbitrary objects in a bin) through trial and error. Within this formulation, we investigate the synergies between grasping and throwing (i.e., learning grasps that enable more accurate throws) and between simulation and deep learning (i.e., using deep networks to predict residuals on top of control parameters predicted by a physics simulator). The resulting system, TossingBot, is able to grasp and throw arbitrary objects into boxes located outside its maximum reach range at 500+ mean picks per hour (600+ grasps per hour with 85% throwing accuracy); and generalizes to new objects and target locations. Videos are available at https://tossingbot.cs.princeton.edu},
  date        = {2019-03-27},
  eprint      = {1903.11239},
  eprintclass = {cs.RO},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1903.11239v3:PDF},
  keywords    = {cs.RO, cs.AI, cs.CV, cs.LG, stat.ML},
}

@Book{Sutton2018,
  author    = {Richard Sutton and Andrew Barto},
  publisher = {MIT Press Ltd},
  title     = {{Reinforcement Learning}},
  year      = {2018},
  isbn      = {0262039249},
  date      = {2018-11-13},
  ean       = {9780262039246},
  pagetotal = {552},
  url       = {https://www.ebook.de/de/product/32966850/richard_s_university_of_alberta_sutton_andrew_g_co_director_autonomous_learning_laboratory_barto_reinforcement_learning.html},
}

@Article{Williams1992,
  author    = {Ronald J. Williams},
  journal   = {Machine Learning},
  title     = {Simple statistical gradient-following algorithms for connectionist reinforcement learning},
  year      = {1992},
  month     = {may},
  number    = {3-4},
  pages     = {229--256},
  volume    = {8},
  doi       = {10.1007/bf00992696},
  publisher = {Springer Science and Business Media {LLC}},
}

@Article{Baxter2011,
  author       = {Jonathan Baxter and Peter L. Bartlett},
  journal      = {{}},
  title        = {Infinite-Horizon Policy-Gradient Estimation},
  year         = {2011},
  abstract     = {Gradient-based approaches to direct policy search in reinforcement learning have received much recent attention as a means to solve problems of partial observability and to avoid some of the problems associated with policy degradation in value-function methods. In this paper we introduce GPOMDP, a simulation-based algorithm for generating a {\em biased} estimate of the gradient of the {\em average reward} in Partially Observable Markov Decision Processes (POMDPs) controlled by parameterized stochastic policies. A similar algorithm was proposed by Kimura, Yamamura, and Kobayashi (1995). The algorithm's chief advantages are that it requires storage of only twice the number of policy parameters, uses one free parameter $\beta\in [0,1)$ (which has a natural interpretation in terms of bias-variance trade-off), and requires no knowledge of the underlying state. We prove convergence of GPOMDP, and show how the correct choice of the parameter $\beta$ is related to the {\em mixing time} of the controlled POMDP. We briefly describe extensions of GPOMDP to controlled Markov chains, continuous state, observation and control spaces, multiple-agents, higher-order derivatives, and a version for training stochastic policies with internal states. In a companion paper (Baxter, Bartlett, & Weaver, 2001) we show how the gradient estimates generated by GPOMDP can be used in both a traditional stochastic gradient algorithm and a conjugate-gradient procedure to find local optima of the average reward},
  date         = {2011-06-03},
  doi          = {10.1613/jair.806},
  eprint       = {1106.0665},
  eprintclass  = {cs.AI},
  eprinttype   = {arXiv},
  file         = {:http\://arxiv.org/pdf/1106.0665v2:PDF},
  journaltitle = {Journal Of Artificial Intelligence Research, Volume 15, pages 319-350, 2001},
  keywords     = {cs.AI},
}

@InProceedings{pmlr-v28-levine13,
  author    = {Sergey Levine and Vladlen Koltun},
  booktitle = {Proceedings of the 30th International Conference on Machine Learning},
  title     = {{Guided policy search}},
  year      = {2013},
  address   = {Atlanta, Georgia, USA},
  editor    = {Sanjoy Dasgupta and David McAllester},
  month     = {17--19 Jun},
  pages     = {1--9},
  publisher = {PMLR},
  series    = {Proceedings of Machine Learning Research},
  volume    = {28},
  abstract  = {Direct policy search can effectively scale to high-dimensional systems, but complex policies with hundreds of parameters often present a challenge for such methods, requiring numerous samples and often falling into poor local optima. We present a guided policy search algorithm that uses trajectory optimization to direct policy learning and avoid poor local optima. We show how differential dynamic programming can be used to generate suitable guiding samples, and describe a regularized importance sampled policy optimization that incorporates these samples into the policy search. We evaluate the method by learning neural network controllers for planar swimming, hopping, and walking, as well as simulated 3D humanoid running.},
  pdf       = {http://proceedings.mlr.press/v28/levine13.pdf},
  url       = {http://proceedings.mlr.press/v28/levine13.html},
}

@Article{Hasselt2015,
  author      = {Hado van Hasselt and Arthur Guez and David Silver},
  journal     = {{}},
  title       = {Deep reinforcement learning with double q-learning},
  year        = {2015},
  abstract    = {The popular Q-learning algorithm is known to overestimate action values under certain conditions. It was not previously known whether, in practice, such overestimations are common, whether they harm performance, and whether they can generally be prevented. In this paper, we answer all these questions affirmatively. In particular, we first show that the recent DQN algorithm, which combines Q-learning with a deep neural network, suffers from substantial overestimations in some games in the Atari 2600 domain. We then show that the idea behind the Double Q-learning algorithm, which was introduced in a tabular setting, can be generalized to work with large-scale function approximation. We propose a specific adaptation to the DQN algorithm and show that the resulting algorithm not only reduces the observed overestimations, as hypothesized, but that this also leads to much better performance on several games.},
  date        = {2015-09-22},
  eprint      = {1509.06461},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1509.06461v3:PDF},
  keywords    = {cs.LG},
}

@Article{Mnih2013,
  author      = {Volodymyr Mnih and Koray Kavukcuoglu and David Silver and Alex Graves and Ioannis Antonoglou and Daan Wierstra and Martin Riedmiller},
  journal     = {{}},
  title       = {Playing Atari with deep reinforcement learning},
  year        = {2013},
  abstract    = {We present the first deep learning model to successfully learn control policies directly from high-dimensional sensory input using reinforcement learning. The model is a convolutional neural network, trained with a variant of Q-learning, whose input is raw pixels and whose output is a value function estimating future rewards. We apply our method to seven Atari 2600 games from the Arcade Learning Environment, with no adjustment of the architecture or learning algorithm. We find that it outperforms all previous approaches on six of the games and surpasses a human expert on three of them.},
  date        = {2013-12-19},
  eprint      = {1312.5602},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1312.5602v1:PDF},
  keywords    = {cs.LG},
}

@Article{Lillicrap2015,
  author      = {Timothy P. Lillicrap and Jonathan J. Hunt and Alexander Pritzel and Nicolas Heess and Tom Erez and Yuval Tassa and David Silver and Daan Wierstra},
  journal     = {{}},
  title       = {Continuous control with deep reinforcement learning},
  year        = {2015},
  abstract    = {We adapt the ideas underlying the success of Deep Q-Learning to the continuous action domain. We present an actor-critic, model-free algorithm based on the deterministic policy gradient that can operate over continuous action spaces. Using the same learning algorithm, network architecture and hyper-parameters, our algorithm robustly solves more than 20 simulated physics tasks, including classic problems such as cartpole swing-up, dexterous manipulation, legged locomotion and car driving. Our algorithm is able to find policies whose performance is competitive with those found by a planning algorithm with full access to the dynamics of the domain and its derivatives. We further demonstrate that for many of the tasks the algorithm can learn policies end-to-end: directly from raw pixel inputs.},
  date        = {2015-09-09},
  eprint      = {1509.02971},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1509.02971v6:PDF},
  keywords    = {cs.LG, stat.ML},
}

@Article{Gu2016,
  author      = {Shixiang Gu and Timothy Lillicrap and Ilya Sutskever and Sergey Levine},
  journal     = {{}},
  title       = {{Continuous deep q-learning with model-based acceleration}},
  year        = {2016},
  abstract    = {Model-free reinforcement learning has been successfully applied to a range of challenging problems, and has recently been extended to handle large neural network policies and value functions. However, the sample complexity of model-free algorithms, particularly when using high-dimensional function approximators, tends to limit their applicability to physical systems. In this paper, we explore algorithms and representations to reduce the sample complexity of deep reinforcement learning for continuous control tasks. We propose two complementary techniques for improving the efficiency of such algorithms. First, we derive a continuous variant of the Q-learning algorithm, which we call normalized adantage functions (NAF), as an alternative to the more commonly used policy gradient and actor-critic methods. NAF representation allows us to apply Q-learning with experience replay to continuous tasks, and substantially improves performance on a set of simulated robotic control tasks. To further improve the efficiency of our approach, we explore the use of learned models for accelerating model-free reinforcement learning. We show that iteratively refitted local linear models are especially effective for this, and demonstrate substantially faster learning on domains where such models are applicable.},
  date        = {2016-03-02},
  eprint      = {1603.00748},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1603.00748v1:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, cs.SY},
}

@Article{Wang2015,
  author      = {Ziyu Wang and Tom Schaul and Matteo Hessel and Hado van Hasselt and Marc Lanctot and Nando de Freitas},
  journal     = {{}},
  title       = {Dueling network architectures for deep reinforcement learning},
  year        = {2015},
  abstract    = {In recent years there have been many successes of using deep representations in reinforcement learning. Still, many of these applications use conventional architectures, such as convolutional networks, LSTMs, or auto-encoders. In this paper, we present a new neural network architecture for model-free reinforcement learning. Our dueling network represents two separate estimators: one for the state value function and one for the state-dependent action advantage function. The main benefit of this factoring is to generalize learning across actions without imposing any change to the underlying reinforcement learning algorithm. Our results show that this architecture leads to better policy evaluation in the presence of many similar-valued actions. Moreover, the dueling architecture enables our RL agent to outperform the state-of-the-art on the Atari 2600 domain.},
  date        = {2015-11-20},
  eprint      = {1511.06581},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1511.06581v3:PDF},
  keywords    = {cs.LG},
}

@Article{Szepesvari2010,
  author    = {Csaba Szepesv{\'{a}}ri},
  journal   = {Synthesis Lectures on Artificial Intelligence and Machine Learning},
  title     = {Algorithms for Reinforcement Learning},
  year      = {2010},
  month     = {jan},
  number    = {1},
  pages     = {1--103},
  volume    = {4},
  doi       = {10.2200/s00268ed1v01y201005aim009},
  publisher = {Morgan {\&} Claypool Publishers {LLC}},
}

@Article{Hanuka2020,
  author      = {Adi Hanuka and X. Huang and J. Shtalenkova and D. Kennedy and A. Edelen and V. R. Lalchand and D. Ratner and J. Duris},
  journal     = {{}},
  title       = {Physics-informed Gaussian Process for Online Optimization of Particle Accelerators},
  year        = {2020},
  abstract    = {High-dimensional optimization is a critical challenge for operating large-scale scientific facilities. We apply a physics-informed Gaussian process (GP) optimizer to tune a complex system by conducting efficient global search. Typical GP models learn from past observations to make predictions, but this reduces their applicability to new systems where archive data is not available. Instead, here we use a fast approximate model from physics simulations to design the GP model. The GP is then employed to make inferences from sequential online observations in order to optimize the system. Simulation and experimental studies were carried out to demonstrate the method for online control of a storage ring. We show that the physics-informed GP outperforms current routinely used online optimizers in terms of convergence speed, and robustness on this task. The ability to inform the machine-learning model with physics may have wide applications in science.},
  date        = {2020-09-08},
  eprint      = {2009.03566},
  eprintclass = {physics.comp-ph},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2009.03566v1:PDF},
  keywords    = {physics.comp-ph, cs.LG, physics.acc-ph},
}

@Article{Roussel2020,
  author      = {Ryan Roussel and Adi Hanuka and Auralee Edelen},
  journal     = {{}},
  title       = {Multi-Objective Bayesian Optimization for Accelerator Tuning},
  year        = {2020},
  abstract    = {Particle accelerators require constant tuning during operation to meet beam quality, total charge and particle energy requirements for use in a wide variety of physics, chemistry and biology experiments. Maximizing the performance of an accelerator facility often necessitates multi-objective optimization, where operators must balance trade-offs between multiple objectives simultaneously, often using limited, temporally expensive beam observations. Usually, accelerator optimization problems are solved offline, prior to actual operation, with advanced beamline simulations and parallelized optimization methods (NSGA-II, Swarm Optimization). Unfortunately, it is not feasible to use these methods for online multi-objective optimization, since beam measurements can only be done in a serial fashion, and these optimization methods require a large number of measurements to converge to a useful solution.Here, we introduce a multi-objective Bayesian optimization scheme, which finds the full Pareto front of an accelerator optimization problem efficiently in a serialized manner and is thus a critical step towards practical online multi-objective optimization in accelerators.This method uses a set of Gaussian process surrogate models, along with a multi-objective acquisition function, which reduces the number of observations needed to converge by at least an order of magnitude over current methods.We demonstrate how this method can be modified to specifically solve optimization challenges posed by the tuning of accelerators.This includes the addition of optimization constraints, objective preferences and costs related to changing accelerator parameters.},
  date        = {2020-10-19},
  eprint      = {2010.09824},
  eprintclass = {physics.acc-ph},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2010.09824v1:PDF},
  keywords    = {physics.acc-ph, physics.comp-ph},
}

@Article{Huang2013,
  author    = {Xiaobiao Huang and Jeff Corbett and James Safranek and Juhao Wu},
  journal   = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment},
  title     = {An algorithm for online optimization of accelerators},
  year      = {2013},
  month     = {oct},
  pages     = {77--83},
  volume    = {726},
  doi       = {10.1016/j.nima.2013.05.046},
  publisher = {Elsevier {BV}},
}

@Article{Bruchon2020,
  author    = {Niky Bruchon and Gianfranco Fenu and Giulio Gaio and Marco Lonza and Finn Henry O'Shea and Felice Andrea Pellegrino and Erica Salvato},
  journal   = {Electronics},
  title     = {Basic reinforcement learning techniques to control the intensity of a seeded free-electron laser},
  year      = {2020},
  month     = {may},
  number    = {5},
  pages     = {781},
  volume    = {9},
  doi       = {10.3390/electronics9050781},
  publisher = {{MDPI} {AG}},
}

@InProceedings{Bruchon2019,
  author    = {Niky Bruchon and Gianfranco Fenu and Giulio Gaio and Marco Lonza and Felice Andrea Pellegrino and Erica Salvato},
  booktitle = {2019 23rd International Conference on Mechatronics Technology ({ICMT})},
  title     = {Toward the Application of Reinforcement Learning to the Intensity Control of a Seeded Free-Electron Laser},
  year      = {2019},
  month     = {oct},
  publisher = {{IEEE}},
  doi       = {10.1109/icmect.2019.8932150},
}

@Article{Bruchon2017,
  author    = {Niky Bruchon and Gianfranco Fenu and Giulio Gaio and Marco Lonza and Felice Andrea Pellegrino and Lorenzo Saule},
  journal   = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment},
  title     = {Free-electron laser spectrum evaluation and automatic optimization},
  year      = {2017},
  month     = {nov},
  pages     = {20--29},
  volume    = {871},
  doi       = {10.1016/j.nima.2017.07.048},
  publisher = {Elsevier {BV}},
}

@Article{Pang2020,
  author      = {Xiaoying Pang and Sunil Thulasidasan and Larry Rybarcyk},
  journal     = {{}},
  title       = {Autonomous Control of a Particle Accelerator using Deep Reinforcement Learning},
  year        = {2020},
  abstract    = {We describe an approach to learning optimal control policies for a large, linear particle accelerator using deep reinforcement learning coupled with a high-fidelity physics engine. The framework consists of an AI controller that uses deep neural nets for state and action-space representation and learns optimal policies using reward signals that are provided by the physics simulator. For this work, we only focus on controlling a small section of the entire accelerator. Nevertheless, initial results indicate that we can achieve better-than-human level performance in terms of particle beam current and distribution. The ultimate goal of this line of work is to substantially reduce the tuning time for such facilities by orders of magnitude, and achieve near-autonomous control.},
  date        = {2020-10-16},
  eprint      = {2010.08141},
  eprintclass = {cs.AI},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2010.08141v1:PDF},
  keywords    = {cs.AI, cs.LG, physics.acc-ph},
}

@Article{Hirlaender2019,
  author    = {Hirlaender, Simon and Fraser, Matthew and Goddard, Brennan and Kain, Verena and Prieto, Javier and Stoel, Linda and Szakaly, Marcel and Velotti, Francesco},
  journal   = {Proceedings of the 10th Int. Particle Accelerator Conf.},
  title     = {{Automatisation of the SPS ElectroStatic septa alignment}},
  year      = {2019},
  pages     = {Australia},
  volume    = {IPAC2019},
  keywords  = {Accelerator Physics, MC6: Beam Instrumentation, Controls, Feedback and Operational Aspects},
  publisher = {JACoW Publishing, Geneva, Switzerland},
}

@Article{Welsch2015,
  author    = {Welsch, Carsten},
  journal   = {Proceedings of the 6th Int. Particle Accelerator Conf.},
  title     = {Numerical Optimization of Accelerators within oPAC},
  year      = {2015},
  pages     = {USA},
  volume    = {IPAC2015},
  keywords  = {Accelerator Physics, 5: Beam Dynamics and EM Fields},
  publisher = {JACoW, Geneva, Switzerland},
}

@Article{6654139,
  author  = {M. P. {Deisenroth} and D. {Fox} and C. E. {Rasmussen}},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence},
  title   = {Gaussian Processes for Data-Efficient Learning in Robotics and Control},
  year    = {2015},
  number  = {2},
  pages   = {408-423},
  volume  = {37},
  doi     = {10.1109/TPAMI.2013.218},
}

@Article{Chua2018,
  author      = {Kurtland Chua and Roberto Calandra and Rowan McAllister and Sergey Levine},
  journal     = {{}},
  title       = {Deep Reinforcement Learning in a Handful of Trials using Probabilistic Dynamics Models},
  year        = {2018},
  abstract    = {Model-based reinforcement learning (RL) algorithms can attain excellent sample efficiency, but often lag behind the best model-free algorithms in terms of asymptotic performance. This is especially true with high-capacity parametric function approximators, such as deep networks. In this paper, we study how to bridge this gap, by employing uncertainty-aware dynamics models. We propose a new algorithm called probabilistic ensembles with trajectory sampling (PETS) that combines uncertainty-aware deep network dynamics models with sampling-based uncertainty propagation. Our comparison to state-of-the-art model-based and model-free deep RL algorithms shows that our approach matches the asymptotic performance of model-free algorithms on several challenging benchmark tasks, while requiring significantly fewer samples (e.g., 8 and 125 times fewer samples than Soft Actor Critic and Proximal Policy Optimization respectively on the half-cheetah task).},
  date        = {2018-05-30},
  eprint      = {1805.12114},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1805.12114v2:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@Article{Wang2019a,
  author      = {Tingwu Wang and Jimmy Ba},
  journal     = {{}},
  title       = {Exploring Model-based Planning with Policy Networks},
  year        = {2019},
  abstract    = {Model-based reinforcement learning (MBRL) with model-predictive control or online planning has shown great potential for locomotion control tasks in terms of both sample efficiency and asymptotic performance. Despite their initial successes, the existing planning methods search from candidate sequences randomly generated in the action space, which is inefficient in complex high-dimensional environments. In this paper, we propose a novel MBRL algorithm, model-based policy planning (POPLIN), that combines policy networks with online planning. More specifically, we formulate action planning at each time-step as an optimization problem using neural networks. We experiment with both optimization w.r.t. the action sequences initialized from the policy network, and also online optimization directly w.r.t. the parameters of the policy network. We show that POPLIN obtains state-of-the-art performance in the MuJoCo benchmarking environments, being about 3x more sample efficient than the state-of-the-art algorithms, such as PETS, TD3 and SAC. To explain the effectiveness of our algorithm, we show that the optimization surface in parameter space is smoother than in action space. Further more, we found the distilled policy network can be effectively applied without the expansive model predictive control during test time for some environments such as Cheetah. Code is released in https://github.com/WilsonWangTHU/POPLIN.},
  date        = {2019-06-20},
  eprint      = {1906.08649},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1906.08649v1:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@Article{Janner2019,
  author      = {Michael Janner and Justin Fu and Marvin Zhang and Sergey Levine},
  journal     = {{}},
  title       = {When to Trust Your Model: Model-Based Policy Optimization},
  year        = {2019},
  abstract    = {Designing effective model-based reinforcement learning algorithms is difficult because the ease of data generation must be weighed against the bias of model-generated data. In this paper, we study the role of model usage in policy optimization both theoretically and empirically. We first formulate and analyze a model-based reinforcement learning algorithm with a guarantee of monotonic improvement at each step. In practice, this analysis is overly pessimistic and suggests that real off-policy data is always preferable to model-generated on-policy data, but we show that an empirical estimate of model generalization can be incorporated into such analysis to justify model usage. Motivated by this analysis, we then demonstrate that a simple procedure of using short model-generated rollouts branched from real data has the benefits of more complicated model-based algorithms without the usual pitfalls. In particular, this approach surpasses the sample efficiency of prior model-based methods, matches the asymptotic performance of the best model-free algorithms, and scales to horizons that cause other model-based methods to fail entirely.},
  date        = {2019-06-19},
  eprint      = {1906.08253},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1906.08253v2:PDF},
  keywords    = {cs.LG, cs.AI, stat.ML},
}

@Misc{Hill2018,
  author       = {Hill, Ashley and Raffin, Antonin and Ernestus, Maximilian and Gleave, Adam and Kanervisto, Anssi and Traore, Rene and Dhariwal, Prafulla and Hesse, Christopher and Klimov, Oleg and Nichol, Alex and Plappert, Matthias and Radford, Alec and Schulman, John and Sidor, Szymon and Wu, Yuhuai},
  howpublished = {\url{https://github.com/hill-a/stable-baselines}},
  title        = {Stable Baselines},
  year         = {2018},
  journal      = {GitHub repository},
  publisher    = {GitHub},
}

@Article{Gu2007,
  author    = {Yaqing Gu and Dean S. Oliver},
  journal   = {{SPE} Journal},
  title     = {An Iterative Ensemble Kalman Filter for Multiphase Fluid Flow Data Assimilation},
  year      = {2007},
  month     = {nov},
  number    = {04},
  pages     = {438--446},
  volume    = {12},
  doi       = {10.2118/108438-pa},
  publisher = {Society of Petroleum Engineers ({SPE})},
}

@Article{Chen2011,
  author    = {Yan Chen and Dean S. Oliver},
  journal   = {Mathematical Geosciences},
  title     = {Ensemble Randomized Maximum Likelihood Method as an Iterative Ensemble Smoother},
  year      = {2011},
  month     = {dec},
  number    = {1},
  pages     = {1--26},
  volume    = {44},
  doi       = {10.1007/s11004-011-9376-z},
  publisher = {Springer Science and Business Media {LLC}},
}

@Article{Bardsley2012,
  author    = {Johnathan M. Bardsley},
  journal   = {{SIAM} Journal on Scientific Computing},
  title     = {{MCMC}-Based Image Reconstruction with Uncertainty Quantification},
  year      = {2012},
  month     = {jan},
  number    = {3},
  pages     = {A1316--A1332},
  volume    = {34},
  doi       = {10.1137/11085760x},
  publisher = {Society for Industrial {\&} Applied Mathematics ({SIAM})},
}

@Article{Albright2019,
  author       = {Albright, Simon and Alemany Fernandez, Reyes and Angoletta, Maria Elena and Bartosik, Hannes and Beaumont, Anthony and Bellodi, Giulia and Biancacci, Nicolo and Bozzolan, Michele and Buzio, Marco and Di Lorenzo, Francesco and Frassier, Alexandre and Gamba, Davide and Hirlander, Simon and Huschauer, Alexander and Kain, Verena and Kotzian, Gerd and Kuchler, Detlef and Latina, Andrea and Levens, Tom and Mahner, Edgar and Manosperti, Enrico and Marqversen, Ole and Moreno Garcia, Daniel and Nicosia, Dom and O'Neil, Michael and Ozturk, Emin and Saa Hernandez, Angela and Scrivens, Richard and Jensen, Steen and Tranquille, Gerard Alain and Wetton, Chris and Zampetakis, Michail},
  journal      = {{}},
  title        = {{Review of LEIR operation in 2018}},
  year         = {2019},
  month        = dec,
  reportnumber = {CERN-ACC-NOTE-2020-0023},
  url          = {http://cds.cern.ch/record/2715365},
}

@Article{Boer2005,
  author    = {Pieter-Tjerk de Boer and Dirk P. Kroese and Shie Mannor and Reuven Y. Rubinstein},
  journal   = {Annals of Operations Research},
  title     = {A Tutorial on the Cross-Entropy Method},
  year      = {2005},
  month     = {feb},
  number    = {1},
  pages     = {19--67},
  volume    = {134},
  doi       = {10.1007/s10479-005-5724-z},
  publisher = {Springer Science and Business Media {LLC}},
}

@Book{Goodfellow2016,
  author    = {Ian Goodfellow and Yoshua Bengio and Aaron Courville},
  publisher = {MIT Press},
  title     = {Deep Learning},
  year      = {2016},
  note      = {\url{http://www.deeplearningbook.org}},
}

@Article{Kain2020,
  author    = {Verena Kain and Simon Hirlander and Brennan Goddard and Francesco Maria Velotti and Giovanni Zevi Della Porta and Niky Bruchon and Gianluca Valentino},
  journal   = {Physical Review Accelerators and Beams},
  title     = {Sample-efficient reinforcement learning for {CERN} accelerator control},
  year      = {2020},
  month     = {dec},
  number    = {12},
  volume    = {23},
  doi       = {10.1103/physrevaccelbeams.23.124801},
  publisher = {American Physical Society ({APS})},
}

@Article{DulacArnold2019,
  author      = {Gabriel Dulac-Arnold and Daniel Mankowitz and Todd Hester},
  journal     = {{}},
  title       = {Challenges of Real-World Reinforcement Learning},
  year        = {2019},
  abstract    = {Reinforcement learning (RL) has proven its worth in a series of artificial domains, and is beginning to show some successes in real-world scenarios. However, much of the research advances in RL are often hard to leverage in real-world systems due to a series of assumptions that are rarely satisfied in practice. We present a set of nine unique challenges that must be addressed to productionize RL to real world problems. For each of these challenges, we specify the exact meaning of the challenge, present some approaches from the literature, and specify some metrics for evaluating that challenge. An approach that addresses all nine challenges would be applicable to a large number of real world problems. We also present an example domain that has been modified to present these challenges as a testbed for practical RL research.},
  date        = {2019-04-29},
  eprint      = {1904.12901},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1904.12901v1:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@Article{Scheinker2020,
  author    = {Alexander Scheinker and Simon Hirlaender and Francesco Maria Velotti and Spencer Gessner and Giovanni Zevi Della Porta and Verena Kain and Brennan Goddard and Rebecca Ramjiawan},
  journal   = {{AIP} Advances},
  title     = {Online multi-objective particle accelerator optimization of the {AWAKE} electron beam line for simultaneous emittance and orbit control},
  year      = {2020},
  month     = {may},
  number    = {5},
  pages     = {055320},
  volume    = {10},
  doi       = {10.1063/5.0003423},
  publisher = {{AIP} Publishing},
}

@Article{Scheinker2018,
  author    = {Alexander Scheinker and Auralee Edelen and Dorian Bohler and Claudio Emma and Alberto Lutman},
  journal   = {Physical Review Letters},
  title     = {Demonstration of Model-Independent Control of the Longitudinal Phase Space of Electron Beams in the Linac-Coherent Light Source with Femtosecond Resolution},
  year      = {2018},
  month     = {jul},
  number    = {4},
  volume    = {121},
  doi       = {10.1103/physrevlett.121.044801},
  publisher = {American Physical Society ({APS})},
}

@Article{Scheinker2019,
  author    = {Alexander Scheinker and Dorian Bohler and Sergey Tomin and Raimund Kammering and Igor Zagorodnov and Holger Schlarb and Matthias Scholz and Bolko Beutner and Winfried Decking},
  journal   = {Physical Review Accelerators and Beams},
  title     = {Model-independent tuning for maximizing free electron laser pulse energy},
  year      = {2019},
  month     = {aug},
  number    = {8},
  volume    = {22},
  doi       = {10.1103/physrevaccelbeams.22.082802},
  publisher = {American Physical Society ({APS})},
}

@Article{Heess2017,
  author      = {Nicolas Heess and Dhruva TB and Srinivasan Sriram and Jay Lemmon and Josh Merel and Greg Wayne and Yuval Tassa and Tom Erez and Ziyu Wang and S. M. Ali Eslami and Martin Riedmiller and David Silver},
  journal     = {{}},
  title       = {Emergence of Locomotion Behaviours in Rich Environments},
  year        = {2017},
  abstract    = {The reinforcement learning paradigm allows, in principle, for complex behaviours to be learned directly from simple reward signals. In practice, however, it is common to carefully hand-design the reward function to encourage a particular solution, or to derive it from demonstration data. In this paper explore how a rich environment can help to promote the learning of complex behavior. Specifically, we train agents in diverse environmental contexts, and find that this encourages the emergence of robust behaviours that perform well across a suite of tasks. We demonstrate this principle for locomotion -- behaviours that are known for their sensitivity to the choice of reward. We train several simulated bodies on a diverse set of challenging terrains and obstacles, using a simple reward function based on forward progress. Using a novel scalable variant of policy gradient reinforcement learning, our agents learn to run, jump, crouch and turn as required by the environment without explicit reward-based guidance. A visual depiction of highlights of the learned behavior can be viewed following https://youtu.be/hx_bgoTF7bs .},
  date        = {2017-07-07},
  eprint      = {1707.02286},
  eprintclass = {cs.AI},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1707.02286v2:PDF},
  keywords    = {cs.AI},
}

@Article{OpenAI2018,
  author      = {OpenAI and Marcin Andrychowicz and Bowen Baker and Maciek Chociej and Rafal Jozefowicz and Bob McGrew and Jakub Pachocki and Arthur Petron and Matthias Plappert and Glenn Powell and Alex Ray and Jonas Schneider and Szymon Sidor and Josh Tobin and Peter Welinder and Lilian Weng and Wojciech Zaremba},
  journal     = {{}},
  title       = {Learning Dexterous In-Hand Manipulation},
  year        = {2018},
  abstract    = {We use reinforcement learning (RL) to learn dexterous in-hand manipulation policies which can perform vision-based object reorientation on a physical Shadow Dexterous Hand. The training is performed in a simulated environment in which we randomize many of the physical properties of the system like friction coefficients and an object's appearance. Our policies transfer to the physical robot despite being trained entirely in simulation. Our method does not rely on any human demonstrations, but many behaviors found in human manipulation emerge naturally, including finger gaiting, multi-finger coordination, and the controlled use of gravity. Our results were obtained using the same distributed RL system that was used to train OpenAI Five. We also include a video of our results: https://youtu.be/jwSbzNHGflM},
  date        = {2018-08-01},
  eprint      = {1808.00177},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1808.00177v5:PDF},
  keywords    = {cs.LG, cs.AI, cs.RO, stat.ML},
}

@InProceedings{Gal2016,
  author    = {Gal, Yarin and McAllister, Rowan and Rasmussen, Carl Edward},
  booktitle = {Data-Efficient Machine Learning workshop, International Conference on Machine Learning},
  title     = {{Improving PILCO with Bayesian neural network dynamics models}},
  year      = {2016},
  pages     = {34},
  volume    = {4},
}

@InProceedings{Deisenroth2011,
  author    = {Deisenroth, Marc Peter and Rasmussen, Carl Edward},
  booktitle = {Proceedings of the 28th International Conference on International Conference on Machine Learning},
  title     = {PILCO: A Model-Based and Data-Efficient Approach to Policy Search},
  year      = {2011},
  address   = {Madison, WI, USA},
  pages     = {465–472},
  publisher = {Omnipress},
  series    = {ICML'11},
  abstract  = {In this paper, we introduce PILCO, a practical, data-efficient model-based policy search method. PILCO reduces model bias, one of the key problems of model-based reinforcement learning, in a principled way. By learning a probabilistic dynamics model and explicitly incorporating model uncertainty into long-term planning, PILCO can cope with very little data and facilitates learning from scratch in only a few trials. Policy evaluation is performed in closed form using state-of-the-art approximate inference. Furthermore, policy gradients are computed analytically for policy improvement. We report unprecedented learning efficiency on challenging and high-dimensional control tasks.},
  isbn      = {9781450306195},
  location  = {Bellevue, Washington, USA},
  numpages  = {8},
}

@InProceedings{kidambi2020morel,
  author       = {Kidambi, Rahul and Rajeswaran, Aravind and Netrapalli, Praneeth and Joachims, Thorsten},
  booktitle    = {NeurIPS 2020},
  title        = {MOReL : model-based offline reinforcement learning},
  year         = {2020},
  month        = {May},
  organization = {ACM},
  abstract     = {In offline reinforcement learning (RL), the goal is to learn a highly rewarding policy based solely on a dataset of historical interactions with the environment. The ability to train RL policies offline can greatly expand the applicability of RL, its data efficiency, and its experimental velocity. Prior work in offline RL has been confined almost exclusively to model-free RL approaches. In this work, we present MOReL, an algorithmic framework for model-based offline RL. This framework consists of two steps: (a) learning a pessimistic MDP (P-MDP) using the offline dataset; and (b) learning a near-optimal policy in this P-MDP. The learned P-MDP has the property that for any policy, the performance in the real environment is approximately lower-bounded by the performance in the P-MDP. This enables it to serve as a good surrogate for purposes of policy evaluation and learning, and overcome common pitfalls of model-based RL like model exploitation. Theoretically, we show that MOReL is minimax optimal (up to log factors) for offline RL. Through experiments, we show that MOReL matches or exceeds state-of-the-art results in widely studied offline RL benchmarks. Moreover, the modular design of MOReL enables future advances in its components (e.g. generative modeling, uncertainty estimation, planning etc.) to directly translate into advances for offline RL.},
  url          = {https://www.microsoft.com/en-us/research/publication/morel-model-based-offline-reinforcement-learning/},
}

@Article{Hirlaender2020b,
  author    = {Hirlaender, Simon and Bruchon, Niky},
  journal   = {{}},
  title     = {{MathPhysSim/FERMI\_RL\_Paper: Initial release}},
  year      = {2020},
  copyright = {MIT License},
  doi       = {10.5281/ZENODO.4271580},
  keywords  = {Reinforcement learning, Accelerator physics, Control theory},
  publisher = {Zenodo},
  url       = {https://zenodo.org/record/4271580},
}

@Article{Haarnoja2018,
  author      = {Tuomas Haarnoja and Aurick Zhou and Pieter Abbeel and Sergey Levine},
  journal     = {{}},
  title       = {Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor},
  year        = {2018},
  abstract    = {Model-free deep reinforcement learning (RL) algorithms have been demonstrated on a range of challenging decision making and control tasks. However, these methods typically suffer from two major challenges: very high sample complexity and brittle convergence properties, which necessitate meticulous hyperparameter tuning. Both of these challenges severely limit the applicability of such methods to complex, real-world domains. In this paper, we propose soft actor-critic, an off-policy actor-critic deep RL algorithm based on the maximum entropy reinforcement learning framework. In this framework, the actor aims to maximize expected reward while also maximizing entropy. That is, to succeed at the task while acting as randomly as possible. Prior deep RL methods based on this framework have been formulated as Q-learning methods. By combining off-policy updates with a stable stochastic actor-critic formulation, our method achieves state-of-the-art performance on a range of continuous control benchmark tasks, outperforming prior on-policy and off-policy methods. Furthermore, we demonstrate that, in contrast to other off-policy algorithms, our approach is very stable, achieving very similar performance across different random seeds.},
  date        = {2018-01-04},
  eprint      = {1801.01290},
  eprintclass = {cs.LG},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/1801.01290v2:PDF},
  keywords    = {cs.LG, cs.AI, stat.ML},
}

@Article{Hirlaender2020a,
  author    = {Simon Hirlaender},
  journal   = {{}},
  title     = {MathPhysSim/PER-NAF: Initial release},
  year      = {2020},
  doi       = {10.5281/zenodo.4271647},
  publisher = {Zenodo},
  url       = {https://doi.org/10.5281/zenodo.4271647},
}

@Article{John2020,
  author      = {Jason St. John and Christian Herwig and Diana Kafkes and William A. Pellico and Gabriel N. Perdue and Andres Quintero-Parra and Brian A. Schupbach and Kiyomi Seiya and Nhan Tran and Javier M. Duarte and Yunzhi Huang and Malachi Schram and Rachael Keller},
  journal     = {{}},
  title       = {Real-time artificial intelligence for accelerator control: a study at the fermilab booster},
  year        = {2020},
  abstract    = {We describe a method for precisely regulating the gradient magnet power supply at the Fermilab Booster accelerator complex using a neural network trained via reinforcement learning. We demonstrate preliminary results by training a surrogate machine-learning model on real accelerator data to emulate the Booster environment, and using this surrogate model in turn to train the neural network for its regulation task. We additionally show how the neural networks to be deployed for control purposes may be compiled to execute on field-programmable gate arrays. This capability is important for operational stability in complicated environments such as an accelerator facility.},
  date        = {2020-11-14},
  eprint      = {2011.07371},
  eprintclass = {physics.acc-ph},
  eprinttype  = {arXiv},
  file        = {:http\://arxiv.org/pdf/2011.07371v2:PDF},
  keywords    = {physics.acc-ph},
}

@PhdThesis{Brochon2020,
  author  = {Bruchon, Niky},
  school  = {{Universit\`a degli Studi di Trieste}},
  title   = {Feasibility Investigation on Several Reinforcement Learning Techniques to Improve the Performance of the FERMI Free-Electron Laser},
  year    = {2020},
  month   = nov,
  note    = {(unpublished)},
  type    = {{PhD thesis}},
  comment = {(unpublished)},
}

@Misc{IDALAB,
  author = {{}},
  title  = {IDA LAB},
  url    = {https://ida-lab.sbg.ac.at/},
}

@Comment{jabref-meta: databaseType:bibtex;}


text:
package flint

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestLinterRequiresProject(t *testing.T) {
	linter := &Linter{}
	summary, err := linter.Run(nil, nil)
	assert.Nil(t, summary)
	assert.NotNil(t, err)
	assert.Equal(t, "Must supply a project", err.Error())
}

func TestLinterReturnsSummary(t *testing.T) {
	linter := &Linter{}
	flags := &Flags{true, true, true, true, true} // Run all
	project := &FakeProject{Flags: flags}
	summary, err := linter.Run(project, flags)
	assert.Nil(t, err)
	if assert.NotNil(t, summary) {
		assert.Equal(t, 0, len(summary.Errors))
	}
}

func TestLinterReportsErrors(t *testing.T) {
	linter := &Linter{}
	flags := &Flags{true, true, true, true, true} // Run all
	results := &Flags{}                           // Return false for each check
	project := &FakeProject{Flags: results}
	assert.Equal(t, results.RunReadme, false)
	summary, err := linter.Run(project, flags)
	assert.Nil(t, err)
	if assert.NotNil(t, summary) {
		assert.Equal(t, 10, len(summary.Errors))
	}
}

type FakeProject struct {
	*Flags
}

func (p *FakeProject) CheckReadme() bool {
	return p.RunReadme
}

func (p *FakeProject) CheckContributing() bool {
	return p.RunContributing
}

func (p *FakeProject) CheckLicense() bool {
	return p.RunLicense
}

func (p *FakeProject) CheckBootstrap() bool {
	return p.RunBootstrap
}

func (p *FakeProject) CheckTestScript() bool {
	return p.RunTestScript
}

branch_name: refs/heads/master

gha_language: Go

extension: go


text:
@book{np,
    author = {Garey, Michael R. and Johnson, David S.},
    title = {Computers and Intractability; A Guide to the Theory of NP-Completeness},
    year = {1990},
    isbn = {0716710455},
    publisher = {W. H. Freeman \& Co.},
    address = {New York, NY, USA},
}


text:
package singlefile

import (
	"context"
	"fmt"
	"io"
)

type ForcedResolver struct {
	Field Circle
}

type ModelMethods struct{}

func (m ModelMethods) NoContext() bool {
	return true
}

func (m ModelMethods) WithContext(_ context.Context) bool {
	return true
}

type Errors struct{}

type Error struct {
	ID string
}

func (Error) ErrorOnRequiredField() (string, error) {
	return "", fmt.Errorf("boom")
}

func (Error) ErrorOnNonRequiredField() (string, error) {
	return "", fmt.Errorf("boom")
}

func (Error) NilOnRequiredField() *string {
	return nil
}

type EmbeddedPointerModel struct {
	*EmbeddedPointer
	ID string
}

type EmbeddedPointer struct {
	Title string
}

type MarshalPanic string

func (m *MarshalPanic) UnmarshalGQL(v interface{}) error {
	panic("BOOM")
}

func (m MarshalPanic) MarshalGQL(w io.Writer) {
	panic("BOOM")
}

type Panics struct{}

func (p *Panics) FieldFuncMarshal(ctx context.Context, u []MarshalPanic) []MarshalPanic {
	return []MarshalPanic{MarshalPanic("aa"), MarshalPanic("bb")}
}

type Autobind struct {
	Int   int
	Int32 int32
	Int64 int64

	IdStr string
	IdInt int
}

type OverlappingFields struct {
	Foo    int
	NewFoo int
}

type ObjectDirectivesWithCustomGoModel struct {
	NullableText string // not *string, but schema is `String @toNull` type.
}

type FallbackToStringEncoding string

const (
	FallbackToStringEncodingA FallbackToStringEncoding = "A"
	FallbackToStringEncodingB FallbackToStringEncoding = "B"
	FallbackToStringEncodingC FallbackToStringEncoding = "C"
)

type Primitive int

func (p Primitive) Squared() int {
	return int(p) * int(p)
}

type PrimitiveString string

func (s PrimitiveString) Doubled() string {
	return string(s) + string(s)
}

type Bytes []byte

gha_license_id: MIT


text:
include_directories( ${CURRENT_LIST_DIR} )
LIST( APPEND module_srcs
  ${CURRENT_LIST_DIR}/smemoutput.cpp
  ${CURRENT_LIST_DIR}/mosmemoutput.cpp
)

extension: cmake


text:
@article{stanley_evolving_2002,
	title = {Evolving {Neural} {Networks} through {Augmenting} {Topologies}},
	volume = {10},
	issn = {1063-6560, 1530-9304},
	url = {http://www.mitpressjournals.org/doi/10.1162/106365602320169811},
	doi = {10.1162/106365602320169811},
	abstract = {An important question in neuroevolution is how to gain an advantage from evolving neural network topologies along with weights. We present a method, NeuroEvolution of Augmenting Topologies (NEAT), which outperforms the best ﬁxed-topology method on a challenging benchmark reinforcement learning task. We claim that the increased efﬁciency is due to (1) employing a principled method of crossover of different topologies, (2) protecting structural innovation using speciation, and (3) incrementally growing from minimal structure. We test this claim through a series of ablation studies that demonstrate that each component is necessary to the system as a whole and to each other. What results is signiﬁcantly faster learning. NEAT is also an important contribution to GAs because it shows how it is possible for evolution to both optimize and complexify solutions simultaneously, offering the possibility of evolving increasingly complex solutions over generations, and strengthening the analogy with biological evolution.},
	language = {en},
	number = {2},
	urldate = {2018-03-25},
	journal = {Evolutionary Computation},
	author = {Stanley, Kenneth O. and Miikkulainen, Risto},
	month = jun,
	year = {2002},
	pages = {99--127},
	file = {Stanley and Miikkulainen - 2002 - Evolving Neural Networks through Augmenting Topolo.pdf:/home/manuel/my/.doc/projects/Zotero/storage/7HGMJBM2/Stanley and Miikkulainen - 2002 - Evolving Neural Networks through Augmenting Topolo.pdf:application/pdf}
}

@article{kohl_integrated_2011,
	title = {An {Integrated} {Neuroevolutionary} {Approach} to {Reactive} {Control} and {High}-level {Strategy}},
	url = {http://www.cs.utexas.edu/users/ai-lab/?kohl:ieeetec11},
	journal = {IEEE Transactions on Evolutionary Computation},
	author = {Kohl, Nate and Miikkulainen, Risto},
	year = {2011},
	file = {Kohl and Miikkulainen - 2011 - An Integrated Neuroevolutionary Approach to Reacti.pdf:/home/manuel/my/.doc/projects/Zotero/storage/SHJDDHSL/Kohl and Miikkulainen - 2011 - An Integrated Neuroevolutionary Approach to Reacti.pdf:application/pdf}
}

@article{khandelwal_time_2015,
	series = {International {Conference} on {Computer}, {Communication} and {Convergence} ({ICCC} 2015)},
	title = {Time {Series} {Forecasting} {Using} {Hybrid} {ARIMA} and {ANN} {Models} {Based} on {DWT} {Decomposition}},
	volume = {48},
	issn = {1877-0509},
	url = {http://www.sciencedirect.com/science/article/pii/S1877050915006766},
	doi = {10.1016/j.procs.2015.04.167},
	abstract = {Recently Discrete Wavelet Transform (DWT) has led to a tremendous surge in many domains of science and engineering. In this study, we present the advantage of DWT to improve time series forecasting precision. This article suggests a novel technique of forecasting by segregating a time series dataset into linear and nonlinear components through DWT. At first, DWT is used to decompose the in-sample training dataset of the time series into linear (detailed) and non-linear (approximate) parts. Then, the Autoregressive Integrated Moving Average (ARIMA) and Artificial Neural Network (ANN) models are used to separately recognize and predict the reconstructed detailed and approximate components, respectively. In this manner, the proposed approach tactically utilizes the unique strengths of DWT, ARIMA, and ANN to improve the forecasting accuracy. Our hybrid method is tested on four real-world time series and its forecasting results are compared with those of ARIMA, ANN, and Zhang's hybrid models. Results clearly show that the proposed method achieves best forecasting accuracies for each series.},
	urldate = {2018-10-31},
	journal = {Procedia Computer Science},
	author = {Khandelwal, Ina and Adhikari, Ratnadip and Verma, Ghanshyam},
	month = jan,
	year = {2015},
	keywords = {ARIMA model, Artificial neural network, Discrete wavelet transform, Time serie forecasting, Zhang's hybrid model},
	pages = {173--179},
	file = {ScienceDirect Full Text PDF:/home/manuel/my/.doc/projects/Zotero/storage/FEQVK97A/Khandelwal et al. - 2015 - Time Series Forecasting Using Hybrid ARIMA and ANN.pdf:application/pdf;ScienceDirect Snapshot:/home/manuel/my/.doc/projects/Zotero/storage/PEY26MRQ/S1877050915006766.html:text/html}
}

@article{mehdy_artificial_2017,
	title = {Artificial {Neural} {Networks} in {Image} {Processing} for {Early} {Detection} of {Breast} {Cancer}},
	url = {https://www.hindawi.com/journals/cmmm/2017/2610628/},
	abstract = {Medical imaging techniques have widely been in use in the diagnosis and detection of breast cancer. The drawback of applying these techniques is the large time consumption in the manual diagnosis of each image pattern by a professional radiologist. Automated classifiers could substantially upgrade the diagnosis process, in terms of both accuracy and time requirement by distinguishing benign and malignant patterns automatically. Neural network (NN) plays an important role in this respect, especially in the application of breast cancer detection. Despite the large number of publications that describe the utilization of NN in various medical techniques, only a few reviews are available that guide the development of these algorithms to enhance the detection techniques with respect to specificity and sensitivity. The purpose of this review is to analyze the contents of recently published literature with special attention to techniques and states of the art of NN in medical imaging. We discuss the usage of NN in four different medical imaging applications to show that NN is not restricted to few areas of medicine. Types of NN used, along with the various types of feeding data, have been reviewed. We also address hybrid NN adaptation in breast cancer detection.},
	language = {en},
	urldate = {2018-10-31},
	journal = {Computational and Mathematical Methods in Medicine},
	author = {Mehdy, M. M. and Ng, P. Y. and Shair, E. F. and Saleh, N. I. Md and Gomes, C.},
	year = {2017},
	doi = {10.1155/2017/2610628},
	file = {2610628.pdf:/home/manuel/my/.doc/projects/Zotero/storage/5W452WNH/2610628.pdf:application/pdf;Full Text PDF:/home/manuel/my/.doc/projects/Zotero/storage/7NNDJZNI/Mehdy et al. - 2017 - Artificial Neural Networks in Image Processing for.pdf:application/pdf;Snapshot:/home/manuel/my/.doc/projects/Zotero/storage/CKITJHVU/2610628.html:application/xhtml+xml}
}

@article{risi_neuroevolution_2014,
	title = {Neuroevolution in {Games}: {State} of the {Art} and {Open} {Challenges}},
	shorttitle = {Neuroevolution in {Games}},
	url = {http://arxiv.org/abs/1410.7326},
	abstract = {This paper surveys research on applying neuroevolution (NE) to games. In neuroevolution, artiﬁcial neural networks are trained through evolutionary algorithms, taking inspiration from the way biological brains evolved. We analyse the application of NE in games along ﬁve different axes, which are the role NE is chosen to play in a game, the different types of neural networks used, the way these networks are evolved, how the ﬁtness is determined and what type of input the network receives. The article also highlights important open research challenges in the ﬁeld.},
	language = {en},
	urldate = {2018-10-31},
	journal = {arXiv:1410.7326 [cs]},
	author = {Risi, Sebastian and Togelius, Julian},
	month = oct,
	year = {2014},
	note = {arXiv: 1410.7326},
	keywords = {Computer Science - Neural and Evolutionary Computing},
	annote = {Comment: - Added more references - Corrected typos - Added an overview table (Table 1)},
	file = {Risi and Togelius - 2014 - Neuroevolution in Games State of the Art and Open.pdf:/home/manuel/my/.doc/projects/Zotero/storage/S2QREFAB/Risi and Togelius - 2014 - Neuroevolution in Games State of the Art and Open.pdf:application/pdf}
}

@article{newell_perceptrons._1969,
	title = {Perceptrons. {An} {Introduction} to {Computational} {Geometry}. {Marvin} {Minsky} and {Seymour} {Papert}. {M}.{I}.{T}. {Press}, {Cambridge}, {Mass}., 1969. vi + 258 pp., illus. {Cloth}, \$12; paper, \$4.95},
	volume = {165},
	copyright = {© 1969},
	issn = {0036-8075, 1095-9203},
	url = {http://science.sciencemag.org/content/165/3895/780},
	doi = {10.1126/science.165.3895.780},
	language = {en},
	number = {3895},
	urldate = {2018-11-03},
	journal = {Science},
	author = {Newell, Allen},
	month = aug,
	year = {1969},
	pages = {780--782},
	file = {Snapshot:/home/manuel/my/.doc/projects/Zotero/storage/MIF7FW76/780.html:text/html}
}

repository: Heholord/BachelorThesis

repo_name: Heholord/BachelorThesis


text:
@unpublished{notsoshort,
  title = "{The Not So Short Introduction to LaTeX}",
  author = {Tobias Oetiker},
  note={https://tobi.oetiker.ch/lshort/lshort.pdf},
  year = {2015},
  url={https://tobi.oetiker.ch/lshort/lshort.pdf}
}


text:
@article{theguardian,
  title = {Islamic State claims attacks at Brussels airport and metro station},
  author = {Rankin, Jennifer and Henley, Jon},
  date = {2016-03-22},
  journal = {The Guardian}
}

@article{cnn,
  title = {Here's what we know about the Brussels terror attacks},
  author = {Hume, Time and Ap, Tiffany and Sanchez, Ray},
  date = {2016-03-25},
  journal = {CNN}
}

@online{ddr2,
  title = {Demand Data Repository (DDR2)},
  url = {https://ext.eurocontrol.int/ddr/},
  organization = {EUROCONTROL}
}

repository: polmes/r-ddr

repo_name: polmes/r-ddr

gha_language: R


text:
DESCRIPTION = "Tunneling proxy for all FOG services"
LICENSE = "Apache-2.0"
LIC_FILES_CHKSUM = "file://src/${GO_IMPORT}/LICENSE;md5=86d3f3a95c324c9479bd8986968f4327"

inherit go pkgconfig gitpkgv systemd

PR = "r0"
SRC_URI = "git://git@github.com/PelionIoT/edge-proxy.git;protocol=https;name=ep;depth=1 \
           file://edge-proxy.service \
           file://edge-proxy-watcher.service \
           file://edge-proxy.path \
           file://launch-edge-proxy.sh \
           file://edge-proxy.conf.json \
           "

SYSTEMD_PACKAGES = "${PN}"
SYSTEMD_SERVICE_${PN} = "edge-proxy.service \
edge-proxy.path \
edge-proxy-watcher.service \
"

SYSTEMD_AUTO_ENABLE_${PN} = "enable"

SRCREV_FORMAT = "ep"
SRCREV_ep = "d2e0fcdab1481487cab243c0ed3b4dc41febc49f"
GO_IMPORT = "github.com/PelionIoT/edge-proxy"

RDEPENDS_${PN} = "jq bash"

wbindir = "/wigwag/system/bin"
wetcdir = "/wigwag/etc"
FILES_${PN} = "\
	${wbindir}/edge-proxy\
	${wbindir}/launch-edge-proxy.sh\
	${wetcdir}/edge-proxy.conf.json\
  ${systemd_system_unitdir}/edge-proxy.service\
  ${systemd_system_unitdir}/edge-proxy-watcher.service\
  ${systemd_system_unitdir}/edge-proxy.path\
	"

do_install () {
  install -d ${D}${wbindir}
  install -m 0755 ${B}/${GO_BUILD_BINDIR}/edge-proxy ${D}${wbindir}/
  install -m 0755 ${WORKDIR}/launch-edge-proxy.sh ${D}${wbindir}/
  install -d ${D}${wetcdir}
  install -m 0755 ${WORKDIR}/edge-proxy.conf.json ${D}${wetcdir}/
  install -d ${D}${systemd_system_unitdir}
  install -m 0644 ${WORKDIR}/edge-proxy.service ${D}${systemd_system_unitdir}
  install -m 0644 ${WORKDIR}/edge-proxy-watcher.service ${D}${systemd_system_unitdir}
  install -m 0644 ${WORKDIR}/edge-proxy.path ${D}${systemd_system_unitdir}
}

repository: kjbracey/meta-pelion-edge

repo_name: kjbracey/meta-pelion-edge

extension: bb


text:
package tiktok

import (
	"time"
)

var tickers []ControllableTicker

type ControllableTicker struct {
	C            <-chan time.Time
	c            chan time.Time
	interval     time.Duration
	elapsed      time.Duration
	elapsedChan  chan time.Duration
	shutdownChan chan int
	ticked       int
}

func NewTicker(d time.Duration) ControllableTicker {
	c := make(chan time.Time)
	elapsedChan := make(chan time.Duration)
	shutdownChan := make(chan int)
	ticker := ControllableTicker{
		C:            c,
		interval:     d,
		c:            c,
		elapsedChan:  elapsedChan,
		shutdownChan: shutdownChan,
	}

	tickers = append(tickers, ticker)
	go ticker.listen()
	return ticker
}

func (ticker *ControllableTicker) listen() {
	for {
		select {
		case elapsed := <-ticker.elapsedChan:
			ticker.elapsed += elapsed
			neededTicks := ticker.elapsed / ticker.interval
			for int(neededTicks) > ticker.ticked {
				ticker.ticked++
				ticker.c <- time.Now()
			}
		case <-ticker.shutdownChan:
			return
		}
	}
}

func (ticker *ControllableTicker) Tick(d time.Duration) {
	ticker.elapsedChan <- d
}

func (ticker *ControllableTicker) ShutDown() {
	ticker.shutdownChan <- 0
	removeTicker(ticker)
}

func (ticker ControllableTicker) Stop() {
	ticker.ShutDown()
}

func (ticker ControllableTicker) Chan() <-chan time.Time {
	return ticker.C
}

func removeTicker(ticker *ControllableTicker) {
	for i, present := range tickers {
		if *ticker == present {
			tickers = append(tickers[:i], tickers[i+1:]...)
			return
		}
	}
}

func Tick(d time.Duration) {
	for _, ticker := range tickers {
		ticker.Tick(d)
	}
}

func ClearTickers() {
	tickers = []ControllableTicker{}
}

repository: wjdix/tiktok

repo_name: wjdix/tiktok


text:
SRC_URI = "http://sources.openembedded.org/opie-1.2.5-split_noncore_net_opietooth_lib.tar.bz2;name=split_noncore_net_opietooth_lib"
SRC_URI[split_noncore_net_opietooth_lib.md5sum] = "f70680fddded64a85d9e33526ca328a6"
SRC_URI[split_noncore_net_opietooth_lib.sha256sum] = "f8e356ae914c28e5e4a7388736fee463f6c29d9a0169272e4cd3a15d94106557"
PR = "r0"

require libopietooth1.inc

repository: overo/overo-oe-natty

repo_name: overo/overo-oe-natty

branch_name: HEAD


text:
//go:build !darwin

package swift

func Demangle(input string) (string, error) {
	return input, nil // this is a no-op on non-darwin platforms
}
func DemangleBlob(blob string) string {
	return blob // this is a no-op on non-darwin platforms
}
func DemangleSimple(input string) (string, error) {
	return input, nil // this is a no-op on non-darwin platforms
}
func DemangleSimpleBlob(blob string) string {
	return blob // this is a no-op on non-darwin platforms
}

repository: blacktop/ipsw

repo_name: blacktop/ipsw


text:
@online{ChemgarooMagnetfelder,
  url       = {http://www.chemgapedia.de/vsengine/vlu/vsc/de/ch/11/cmt/sensoren/vlus/magnet_mess.vlu.html},
  author    = {Volker, Schubert and Volker, Rödig},
  year      = {unbekanntes Jahr},
  title     = {Messung von Magnetfeldern - Elektromagnetismus},
  urldate   = {2019-02-13}
}

@online{lokifahrer,
  url       = {http://www.lokifahrer.ch/Lukmanier/Rollmaterial/Bremse/Verschleisslos.htm},
  author    = {unbekannt},
  year      = {2015},
  title     = {Verschleisslose Bremsen},
  urldate   = {2019-02-13}
}

@online{regelungstechnik,
  author    = {Waste},
  title     = {Regelungstechnik},
  date      = {2005-11-16},
  urldate   = {2019-01-30},
  notes     = {'Waste' ist der Benutzername des Authors},
  url       = {https://rn-wissen.de/wiki/index.php?title=Regelungstechnik}
}

@online{drehmoment,
  author    = {kfzteile42},
  title     = {Funktion und Wirkungsweise der Bremse},
  urldate   = {2019-02-09},
  year      = {unbekanntes Jahr},
  url       = {https://www.kfzteile24.de/magazin/lexikon/bremse-aufbau-funktion}
}

@online{scrum_info,
  author    = {Scrum.org},
  title     = {What is Scrum?},
  urldate   = {2019-02-11},
  year      = {unbekanntes Jahr},
  url       = {https://www.scrum.org/resources/what-is-scrum}
}

@article{current_on_a_rotating_disk,
  author  ={W. R. Smythe},
  journal ={Electrical Engineering},
  title   ={On eddy currents in a rotating disk},
  year    ={1942},
  month   ={9},
  volume  ={61},
  number  ={9},
  pages   ={681-684},
  doi     ={10.1109/EE.1942.6436528},
  ISSN    ={0095-9197},
  language  = {Englisch}
}

@article{schulmaterial_regler,
  author  ={Eberli, Stephan},
  journal ={ET4 - Elektronik},
  title   ={Steuern und Regeln, unstetige Regler, stetige Regler, Regler},
  year    ={2018},
  month   ={8},
  pages   ={5-30},
  language  = {Deutsch}
}

@article{schulmaterial_magnetismus,
  author  ={Eberli, Stephan},
  journal ={ET2 - Elektrotechnik},
  title   ={Magnetismus},
  year    ={2017},
  month   ={August},
  pages   ={31-78},
  language  = {Deutsch}
}

@book{bumiller2016fachkunde,
  title     = {Fachkunde Elektrotechnik: bearbeitet von Lehrern an beruflichen Schulen und von Ingenieuren},
  author    = {Bumiller, H. and Tkotz, K. and Burgmaier, M. and Eichler, W. and Feustel, B. and K{\"a}ppel, T. and Klee, W. and Manderla, J. and Reichmann, O. and Schwarz, J. and others},
  isbn      = {9783808534359},
  series    = {Europa-Fachbuchreihe f{\"u}r elektrotechnische Berufe},
  year      = {2016},
  publisher = {Verlag Europa-Lehrmittel}
}

@book{kuchling2014taschenbuch,
  title     ={Taschenbuch der Physik},
  author    ={Kuchling, H.},
  isbn      ={9783446442184},
  year      ={2014},
  publisher ={Hanser Fachbuchverlag}
}


text:
package parser

import (
	"github.com/magiconair/properties"
)

//PropertiesFileParser ...
type PropertiesFileParser struct {
	parserType string
}

//NewPropertiesFileParser ...
func NewPropertiesFileParser() *PropertiesFileParser {
	return &PropertiesFileParser{
		parserType: "properties",
	}
}

//GetParserType ...
func (parser *PropertiesFileParser) GetParserType() string {
	return parser.parserType
}

//Parse ...
func (parser *PropertiesFileParser) Parse(file string) (map[string]interface{}, error) {
	result := map[string]interface{}{}
	prop, err := properties.LoadFile(file, properties.UTF8)
	if err != nil {
		return result, err
	}
	m := prop.Map()
	for key, val := range m {
		result[key] = val
	}
	return result, nil
}

repository: nienie/manyfaced

repo_name: nienie/manyfaced