text:
The results demonstrate on the one hand that the consideration of sample constraints can lead to the discovery of new efficient algorithms for non-trivial learning problems. On the other hand, hard learning problems may remain intractable even for severely restricted samples},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {891–912},
numpages = {22}
}

@article{10.5555/1005332.1016787,
author = {Dy, Jennifer G. and Brodley, Carla E.},
title = {Feature Selection for Unsupervised Learning},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper, we identify two issues involved in developing an automated feature subset selection algorithm for unlabeled data: the need for finding the number of clusters in conjunction with feature selection, and the need for normalizing the bias of feature selection criteria with respect to dimension. We explore the feature selection problem and these issues through FSSEM (Feature Subset Selection using Expectation-Maximization (EM) clustering) and through two different performance criteria for evaluating candidate feature subsets: scatter separability and maximum likelihood. We present proofs on the dimensionality biases of these feature criteria, and present a cross-projection normalization scheme that can be applied to any criterion to ameliorate these biases. Our experiments show the need for feature selection, the need for addressing these two issues, and the effectiveness of our proposed solutions.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {845–889},
numpages = {45}
}

@article{10.5555/1005332.1016786,
author = {Jebara, Tony and Kondor, Risi and Howard, Andrew},
title = {Probability Product Kernels},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {The advantages of discriminative learning algorithms and kernel machines are combined with generative modeling using a novel kernel between distributions. In the probability product kernel, data points in the input space are mapped to distributions over the sample space and a general inner product is then evaluated as the integral of the product of pairs of distributions. The kernel is straightforward to evaluate for all exponential family models such as multinomials and Gaussians and yields interesting nonlinear kernels. Furthermore, the kernel is computable in closed form for latent distributions such as mixture models, hidden Markov models and linear dynamical systems. For intractable models, such as switching linear dynamical systems, structured mean-field approximations can be brought to bear on the kernel evaluation. For general distributions, even if an analytic expression for the kernel is not feasible, we show a straightforward sampling method to evaluate it. Thus, the kernel permits discriminative learning methods, including support vector machines, to exploit the properties, metrics and invariances of the generative models we infer from each datum. Experiments are shown using multinomial models for text, hidden Markov models for biological data sets and linear dynamical systems for time series data.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {819–844},
numpages = {26}
}

@article{10.5555/1005332.1016785,
author = {Laub, Julian and M\"{u}ller, Klaus-Robert},
title = {Feature Discovery in Non-Metric Pairwise Data},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Pairwise proximity data, given as similarity or dissimilarity matrix, can violate metricity. This occurs either due to noise, fallible estimates, or due to intrinsic non-metric features such as they arise from human judgments. So far the problem of non-metric pairwise data has been tackled by essentially omitting the negative eigenvalues or shifting the spectrum of the associated (pseudo-)covariance matrix for a subsequent embedding. However, little attention has been paid to the negative part of the spectrum itself. In particular no answer was given to whether the directions associated to the negative eigenvalues would at all code variance other than noise related. We show by a simple, exploratory analysis that the negative eigenvalues can code for relevant structure in the data, thus leading to the discovery of new features, which were lost by conventional data analysis techniques. The information hidden in the negative eigenvalue part of the spectrum is illustrated and discussed for three data sets, namely USPS handwritten digits, text-mining and data from cognitive psychology.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {801–818},
numpages = {18}
}

@article{10.5555/1005332.1016784,
author = {Ziehe, Andreas and Laskov, Pavel and Nolte, Guido and M\"{u}ller, Klaus-Robert},
title = {A Fast Algorithm for Joint Diagonalization with Non-Orthogonal Transformations and Its Application to Blind Source Separation},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {A new efficient algorithm is presented for joint diagonalization of several matrices. The algorithm is based on the Frobenius-norm formulation of the joint diagonalization problem, and addresses diagonalization with a general, non-orthogonal transformation. The iterative scheme of the algorithm is based on a multiplicative update which ensures the invertibility of the diagonalizer. The algorithm's efficiency stems from the special approximation of the cost function resulting in a sparse, block-diagonal Hessian to be used in the computation of the quasi-Newton update step. Extensive numerical simulations illustrate the performance of the algorithm and provide a comparison to other leading diagonalization methods. The results of such comparison demonstrate that the proposed algorithm is a viable alternative to existing state-of-the-art joint diagonalization algorithms. The practical use of our algorithm is shown for blind source separation problems.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {777–800},
numpages = {24}
}

@article{10.5555/1005332.1016783,
author = {Valentini, Giorgio and Dietterich, Thomas G.},
title = {Bias-Variance Analysis of Support Vector Machines for the Development of SVM-Based Ensemble Methods},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Bias-variance analysis provides a tool to study learning algorithms and can be used to properly design ensemble methods well tuned to the properties of a specific base learner. Indeed the effectiveness of ensemble methods critically depends on accuracy, diversity and learning characteristics of base learners. We present an extended experimental analysis of bias-variance decomposition of the error in Support Vector Machines (SVMs), considering Gaussian, polynomial and dot product kernels. A characterization of the error decomposition is provided, by means of the analysis of the relationships between bias, variance, kernel type and its parameters, offering insights into the way SVMs learn. The results show that the expected trade-off between bias and variance is sometimes observed, but more complex relationships can be detected, especially in Gaussian and polynomial kernels. We show that the bias-variance decomposition offers a rationale to develop ensemble methods using SVMs as base learners, and we outline two directions for developing SVM ensembles, exploiting the SVM bias characteristics and the bias-variance dependence on the kernel param},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {725–775},
numpages = {51}
}

@article{10.5555/1005332.1016782,
author = {Zhang, Nevin L.},
title = {Hierarchical Latent Class Models for Cluster Analysis},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Latent class models are used for cluster analysis of categorical data. Underlying such a model is the assumption that the observed variables are mutually independent given the class variable. A serious problem with the use of latent class models, known as local dependence, is that this assumption is often untrue. In this paper we propose hierarchical latent class models as a framework where the local dependence problem can be addressed in a principled manner. We develop a search-based algorithm for learning hierarchical latent class models from data. The algorithm is evaluated using both synthetic and real-world data.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {697–723},
numpages = {27}
}

@article{10.5555/1005332.1005357,
author = {Luxburg, Ulrike von and Bousquet, Olivier},
title = {Distance--Based Classification with Lipschitz Functions},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {The goal of this article is to develop a framework for large margin classification in metric spaces. We want to find a generalization of linear decision functions for metric spaces and define a corresponding notion of margin such that the decision function separates the training points with a large margin. It will turn out that using Lipschitz functions as decision functions, the inverse of the Lipschitz constant can be interpreted as the size of a margin. In order to construct a clean mathematical setup we isometrically embed the given metric space into a Banach space and the space of Lipschitz functions into its dual space. To analyze the resulting algorithm, we prove several representer theorems. They state that there always exist solutions of the Lipschitz classifier which can be expressed in terms of distance functions to training points. We provide generalization bounds for Lipschitz classifiers in terms of the Rademacher complexities of some Lipschitz function classes. The generality of our approach can be seen from the fact that several well-known algorithms are special cases of the Lipschitz classifier, among them the support vector machine, the linear programming machine, and the 1-nearest neighbor classifier.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {669–695},
numpages = {27}
}

@article{10.5555/1005332.1005356,
author = {Blum, Avrim and Jackson, Jeffrey and Sandholm, Tuomas and Zinkevich, Martin},
title = {Preference Elicitation and Query Learning},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper we explore the relationship between "preference elicitation", a learning-style problem that arises in combinatorial auctions, and the problem of learning via queries studied in computational learning theory. Preference elicitation is the process of asking questions about the preferences of bidders so as to best divide some set of goods. As a learning problem, it can be thought of as a setting in which there are multiple target concepts that can each be queried separately, but where the goal is not so much to learn each concept as it is to produce an "optimal example". In this work, we prove a number of similarities and differences between two-bidder preference elicitation and query learning, giving both separation results and proving some connections between these problems.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {649–667},
numpages = {19}
}

@article{10.5555/1005332.1005355,
author = {Mannor, Shie and Tsitsiklis, John N.},
title = {The Sample Complexity of Exploration in the Multi-Armed Bandit Problem},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We consider the multi-armed bandit problem under the PAC ("probably approximately correct") model. It was shown by Even-Dar et al. (2002) that given n arms, a total of O((n/ε2)log(1/δ)) trials suffices in order to find an ε-optimal arm with probability at least 1-δ. We establish a matching lower bound on the expected number of trials under any sampling policy. We furthermore generalize the lower bound, and show an explicit dependence on the (unknown) statistics of the arms. We also provide a similar bound within a Bayesian setting. The case where the statistics of the arms are known but the identities of the arms are not, is also discussed. For this case, we provide a lower bound of Θ((1/ε2)(n+log(1/δ))) on the expected number of trials, as well as a sampling policy with a matching upper bound. If instead of the expected number of trials, we consider the maximum (over all sample paths) number of trials, we establish a matching upper and lower bound of the form Θ((n/ε2)log(1/δ)). Finally, we derive lower bounds on the expected regret, in the spirit of Lai and Robbins.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {623–648},
numpages = {26}
}

@article{10.5555/1005332.1005354,
author = {Ginter, Filip and Boberg, Jorma and J\"{a}rvinen, Jouni and Salakoski, Tapio},
title = {New Techniques for Disambiguation in Natural Language and Their Application to Biological Text},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We study the problems of disambiguation in natural language, focusing on the problem of gene vs. protein name disambiguation in biological text and also considering the problem of context-sensitive spelling error correction. We introduce a new family of classifiers based on ordering and weighting the feature vectors obtained from word counts and word co-occurrence in the text, and inspect several concrete classifiers from this family. We obtain the most accurate prediction when weighting by positions of the words in the context. On the gene/protein name disambiguation problem, this classifier outperforms both the Naive Bayes and SNoW baseline classifiers. We also study the effect of the smoothing techniques with the Naive Bayes classifier, the collocation features, and the context length on the classification accuracy and show that correct setting of the context length is important and also problem-dependent.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {605–621},
numpages = {17}
}

@article{10.5555/1005332.1005353,
author = {Vovk, Vladimir},
title = {A Universal Well-Calibrated Algorithm for On-Line Classification},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We study the problem of on-line classification in which the prediction algorithm, for each "significance level" δ, is required to output as its prediction a range of labels (intuitively, those labels deemed compatible with the available data at the level δ) rather than just one label; as usual, the examples are assumed to be generated independently from the same probability distribution P. The prediction algorithm is said to be "well-calibrated" for P and δ if the long-run relative frequency of errors does not exceed δ almost surely w.r. to P. For well-calibrated algorithms we take the number of "uncertain" predictions (i.e., those containing more than one label) as the principal measure of predictive performance. The main result of this paper is the construction of a prediction algorithm which, for any (unknown) P and any δ: (a) makes errors independently and with probability δ at every trial (in particular, is well-calibrated for P and δ); (b) makes in the long run no more uncertain predictions than any other prediction algorithm that is well-calibrated for P and δ; (c) processes example n in time O(log n).},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {575–604},
numpages = {30}
}

@article{10.5555/1005332.1005352,
author = {Koivisto, Mikko and Sood, Kismat},
title = {Exact Bayesian Structure Discovery in Bayesian Networks},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Learning a Bayesian network structure from data is a well-motivated but computationally hard task. We present an algorithm that computes the exact posterior probability of a subnetwork, e.g., a directed edge; a modified version of the algorithm finds one of the most probable network structures. This algorithm runs in time O(n 2n + nk+1C(m)), where n is the number of network variables, k is a constant maximum in-degree, and C(m) is the cost of computing a single local marginal conditional likelihood for m data instances. This is the first algorithm with less than super-exponential complexity with respect to n. Exact computation allows us to tackle complex cases where existing Monte Carlo methods and local search procedures potentially fail. We show that also in domains with a large number of variables, exact computation is feasible, given suitable a priori restrictions on the structures; combining exact and inexact methods is also possible. We demonstrate the applicability of the presented algorithm on four synthetic data sets with 17, 22, 37, and 100 variables.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {549–573},
numpages = {25}
}

@article{10.5555/1005332.1005351,
author = {Langford, John and McAllester, David},
title = {Computable Shell Decomposition Bounds},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Haussler, Kearns, Seung and Tishby introduced the notion of a shell decomposition of the union bound as a means of understanding certain empirical phenomena in learning curves such as phase transitions. Here we use a variant of their ideas to derive an upper bound on the generalization error of a hypothesis computable from its training error and the histogram of training errors for the hypotheses in the class. In most cases this new bound is significantly tighter than traditional bounds computed from the training error and the cardinality of the class. Our results can also be viewed as providing a rigorous foundation for a model selection algorithm proposed by Scheffer and Joachims.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {529–547},
numpages = {19}
}

@article{10.5555/1005332.1005350,
author = {Kauchak, David and Smarr, Joseph and Elkan, Charles},
title = {Sources of Success for Boosted Wrapper Induction},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper, we examine an important recent rule-based information extraction (IE) technique named Boosted Wrapper Induction (BWI) by conducting experiments on a wider variety of tasks than previously studied, including tasks using several collections of natural text documents. We investigate systematically how each algorithmic component of BWI, in particular boosting, contributes to its success. We show that the benefit of boosting arises from the ability to reweight examples to learn specific rules (resulting in high precision) combined with the ability to continue learning rules after all positive examples have been covered (resulting in high recall). As a quantitative indicator of the regularity of an extraction task, we propose a new measure that we call the SWI ratio. We show that this measure is a good predictor of IE success and a useful tool for analyzing IE tasks. Based on these results, we analyze the strengths and limitations of BWI. Specifically, we explain limitations in the information made available, and in the representations used. We also investigate the consequences of the fact that confidence values returned during extraction are not true probabilities. Next, we investigate the benefits of including grammatical and semantic information for natural text documents, as well as parse tree and attribute-value information for XML and HTML documents. We show experimentally that incorporating even limited grammatical information can increase the regularity of natural text extraction tasks, resulting in improved performance. We conclude with proposals for enriching the representational power of BWI and other IE methods to exploit these and other types of regularities.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {499–527},
numpages = {29}
}

@article{10.5555/1005332.1005349,
author = {Clark, Alexander and Thollard, Franck},
title = {PAC-Learnability of Probabilistic Deterministic Finite State Automata},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We study the learnability of Probabilistic Deterministic Finite State Automata under a modified PAC-learning criterion. We argue that it is necessary to add additional parameters to the sample complexity polynomial, namely a bound on the expected length of strings generated from any state, and a bound on the distinguishability between states. With this, we demonstrate that the class of PDFAs is PAC-learnable using a variant of a standard state-merging algorithm and the Kullback-Leibler divergence as error function.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {473–497},
numpages = {25}
}

@article{10.5555/1005332.1005348,
author = {Higuchi, Isao and Eguchi, Shinto},
title = {Robust Principal Component Analysis with Adaptive Selection for Tuning Parameters},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {The present paper discusses robustness against outliers in a principal component analysis (PCA). We propose a class of procedures for PCA based on the minimum psi principle, which unifies various approaches, including the classical procedure and recently proposed procedures. The reweighted matrix algorithm for off-line data and the gradient algorithm for on-line data are both investigated with respect to robustness. The reweighted matrix algorithm is shown to satisfy a desirable property with local convergence, and the on-line gradient algorithm is shown to satisfy an asymptotical stability of convergence. Some procedures in the class involve tuning parameters, which control sensitivity to outliers. We propose a shape-adaptive selection rule for tuning parameters using K-fold cross validation.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {453–471},
numpages = {19}
}

@article{10.5555/1005332.1005347,
author = {Chawla, Nitesh V. and Hall, Lawrence O. and Bowyer, Kevin W. and Kegelmeyer, W. Philip},
title = {Learning Ensembles from Bites: A Scalable and Accurate Approach},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Bagging and boosting are two popular ensemble methods that typically achieve better accuracy than a single classifier. These techniques have limitations on massive data sets, because the size of the data set can be a bottleneck. Voting many classifiers built on small subsets of data ("pasting small votes") is a promising approach for learning from massive data sets, one that can utilize the power of boosting and bagging. We propose a framework for building hundreds or thousands of such classifiers on small subsets of data in a distributed environment. Experiments show this approach is fast, accurate, and scalable.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {421–451},
numpages = {31}
}

@article{10.5555/1005332.1005346,
author = {Quist, Michael and Yona, Golan},
title = {Distributional Scaling: An Algorithm for Structure-Preserving Embedding of Metric and Nonmetric Spaces},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We present a novel approach for embedding general metric and nonmetric spaces into low-dimensional Euclidean spaces. As opposed to traditional multidimensional scaling techniques, which minimize the distortion of pairwise distances, our embedding algorithm seeks a low-dimensional representation of the data that preserves the structure (geometry) of the original data. The algorithm uses a hybrid criterion function that combines the pairwise distortion with what we call the geometric distortion. To assess the geometric distortion, we explore functions that reflect geometric properties. Our approach is different from the Isomap and LLE algorithms in that the discrepancy in distributional information is used to guide the embedding. We use clustering algorithms in conjunction with our embedding algorithm to direct the embedding process and improve its convergence properties.We test our method on metric and nonmetric data sets, and in the presence of noise. We demonstrate that our method preserves the structural properties of embedded data better than traditional MDS, and that its performance is robust with respect to clustering errors in the original data. Other results of the paper include accelerated algorithms for optimizing the standard MDS objective functions, and two methods for finding the most appropriate dimension in which to embed a given set of data.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {399–420},
numpages = {22}
}

@article{10.5555/1005332.1005345,
author = {Lewis, David D. and Yang, Yiming and Rose, Tony G. and Li, Fan},
title = {RCV1: A New Benchmark Collection for Text Categorization Research},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Reuters Corpus Volume I (RCV1) is an archive of over 800,000 manually categorized newswire stories recently made available by Reuters, Ltd. for research purposes. Use of this data for research on text categorization requires a detailed understanding of the real world constraints under which the data was produced. Drawing on interviews with Reuters personnel and access to Reuters documentation, we describe the coding policy and quality control procedures used in producing the RCV1 data, the intended semantics of the hierarchical category taxonomies, and the corrections necessary to remove errorful data. We refer to the original data as RCV1-v1, and the corrected data as RCV1-v2. We benchmark several widely used supervised learning methods on RCV1-v2, illustrating the collection's properties, suggesting new directions for research, and providing baseline results for future studies. We make available detailed, per-category experimental results, as well as corrected versions of the category assignments and taxonomy structures, via online appendices.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {361–397},
numpages = {37}
}

@article{10.5555/1005332.1005344,
author = {Mannor, Shie and Shimkin, Nahum},
title = {A Geometric Approach to Multi-Criterion Reinforcement Learning},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We consider the problem of reinforcement learning in a controlled Markov environment with multiple objective functions of the long-term average reward type. The environment is initially unknown, and furthermore may be affected by the actions of other agents, actions that are observed but cannot be predicted beforehand. We capture this situation using a stochastic game model, where the learning agent is facing an adversary whose policy is arbitrary and unknown, and where the reward function is vector-valued. State recurrence conditions are imposed throughout. In our basic problem formulation, a desired target set is specified in the vector reward space, and the objective of the learning agent is to <it>approach</it> the target set, in the sense that the long-term average reward vector will belong to this set. We devise appropriate learning algorithms, that essentially use multiple reinforcement learning algorithms for the standard scalar reward problem, which are combined using the geometric insight from the theory of approachability for vector-valued stochastic games. We then address the more general and optimization-related problem, where a nested class of possible target sets is prescribed, and the goal of the learning agent is to approach the smallest possible target set (which will generally depend on the unknown system parameters). A particular case which falls into this framework is that of stochastic games with average reward constraints, and further specialization provides a reinforcement learning algorithm for constrained Markov decision processes. Some basic examples are provided to illustrate these results.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {325–360},
numpages = {36}
}

@article{10.5555/1005332.1005343,
author = {Luxburg, Ulrike von and Bousquet, Olivier and Sch\"{o}lkopf, Bernhard},
title = {A Compression Approach to Support Vector Model Selection},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper we investigate connections between statistical learning theory and data compression on the basis of support vector machine (SVM) model selection. Inspired by several generalization bounds we construct "compression coefficients" for SVMs which measure the amount by which the training labels can be compressed by a code built from the separating hyperplane. The main idea is to relate the coding precision to geometrical concepts such as the width of the margin or the shape of the data in the feature space. The so derived compression coefficients combine well known quantities such as the radius-margin term R2/ρ2, the eigenvalues of the kernel matrix, and the number of support vectors. To test whether they are useful in practice we ran model selection experiments on benchmark data sets. As a result we found that compression coefficients can fairly accurately predict the parameters for which the test error is minimized.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {293–323},
numpages = {31}
}

@article{10.5555/1005332.1005342,
author = {Baram, Yoram and El-Yaniv, Ran and Luz, Kobi},
title = {Online Choice of Active Learning Algorithms},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {This work is concerned with the question of how to combine online an ensemble of active learners so as to expedite the learning progress in pool-based active learning. We develop an active-learning master algorithm, based on a known competitive algorithm for the multi-armed bandit problem. A major challenge in successfully choosing top performing active learners online is to reliably estimate their progress during the learning session. To this end we propose a simple maximum entropy criterion that provides effective estimates in realistic settings. We study the performance of the proposed master algorithm using an ensemble containing two of the best known active-learning algorithms as well as a new algorithm. The resulting active-learning master algorithm is empirically shown to consistently perform almost as well as and sometimes outperform the best algorithm in the ensemble on a range of classification problems.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {255–291},
numpages = {37}
}

@article{10.5555/1005332.1005341,
author = {Basak, Jayanta and Sudarshan, Anant and Trivedi, Deepak and Santhanam, M. S.},
title = {Weather Data Mining Using Independent Component Analysis},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this article, we apply the independent component analysis technique for mining spatio-temporal data. The technique has been applied to mine for patterns in weather data using the North Atlantic Oscillation (NAO) as a specific example. We find that the strongest independent components match the observed synoptic weather patterns corresponding to the NAO. We also validate our results by matching the independent component activities with the NAO index.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {239–253},
numpages = {15}
}

@article{10.5555/1005332.1005340,
author = {Mendelson, Shahar and Philips, Petra},
title = {On the Importance of Small Coordinate Projections},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {It has been recently shown that sharp generalization bounds can be obtained when the function class from which the algorithm chooses its hypotheses is "small" in the sense that the Rademacher averages of this function class are small. We show that a new more general principle guarantees good generalization bounds. The new principle requires that random coordinate projections of the function class evaluated on random samples are "small" with high probability and that the random class of functions allows symmetrization. As an example, we prove that this geometric property of the function class is exactly the reason why the two lately proposed frameworks, the <it>luckiness</it> (Shawe-Taylor et al., 1998) and the <it>algorithmic luckiness</it> (Herbrich and Williamson, 2002), can be used to establish generalization bounds.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {219–238},
numpages = {20}
}

@article{10.5555/1005332.1005339,
author = {Anthony, Martin},
title = {Generalization Error Bounds for Threshold Decision Lists},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper we consider the generalization accuracy of classification methods based on the iterative use of linear classifiers. The resulting classifiers, which we call threshold decision lists act as follows. Some points of the data set to be classified are given a particular classification according to a linear threshold function (or hyperplane). These are then removed from consideration, and the procedure is iterated until all points are classified. Geometrically, we can imagine that at each stage, points of the same classification are successively chopped off from the data set by a hyperplane. We analyse theoretically the generalization properties of data classification techniques that are based on the use of threshold decision lists and on the special subclass of multilevel threshold functions. We present bounds on the generalization error in a standard probabilistic learning framework. The primary focus in this paper is on obtaining generalization error bounds that depend on the levels of separation---or margins---achieved by the successive linear classifiers. We also improve and extend previously published theoretical bounds on the generalization ability of perceptron decision trees.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {189–217},
numpages = {29}
}

@article{10.5555/1005332.1005338,
author = {Lavra\v{c}, Nada and Kav\v{s}ek, Branko and Flach, Peter and Todorovski, Ljup\v{c}o},
title = {Subgroup Discovery with CN2-SD},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {This paper investigates how to adapt standard classification rule learning approaches to subgroup discovery. The goal of subgroup discovery is to find rules describing subsets of the population that are sufficiently large and statistically unusual. The paper presents a subgroup discovery algorithm, CN2-SD, developed by modifying parts of the CN2 classification rule learner: its covering algorithm, search heuristic, probabilistic classification of instances, and evaluation measures. Experimental evaluation of CN2-SD on 23 UCI data sets shows substantial reduction of the number of induced rules, increased rule coverage and rule significance, as well as slight improvements in terms of the area under ROC curve, when compared with the CN2 algorithm. Application of CN2-SD to a large traffic accident data set confirms these findings.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {153–188},
numpages = {36}
}

@article{10.5555/1005332.1005337,
author = {Lee, Herbert K. H. and Clyde, Merlise A.},
title = {Lossless Online Bayesian Bagging},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Bagging frequently improves the predictive performance of a model. An online version has recently been introduced, which attempts to gain the benefits of an online algorithm while approximating regular bagging. However, regular online bagging is an approximation to its batch counterpart and so is not lossless with respect to the bagging operation. By operating under the Bayesian paradigm, we introduce an online Bayesian version of bagging which is exactly equivalent to the batch Bayesian version, and thus when combined with a lossless learning algorithm gives a completely lossless online bagging algorithm. We also note that the Bayesian formulation resolves a theoretical problem with bagging, produces less variability in its estimates, and can improve predictive performance for smaller data sets.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {143–151},
numpages = {9}
}

@article{10.5555/1005332.1005336,
author = {Rifkin, Ryan and Klautau, Aldebaro},
title = {In Defense of One-Vs-All Classification},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We consider the problem of multiclass classification. Our main thesis is that a simple "one-vs-all" scheme is as accurate as any other approach, assuming that the underlying binary classifiers are well-tuned regularized classifiers such as support vector machines. This thesis is interesting in that it disagrees with a large body of recent published work on multiclass classification. We support our position by means of a critical review of the existing literature, a substantial collection of carefully controlled experimental work, and theoretical arguments.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {101–141},
numpages = {41}
}

@article{10.5555/1005332.1005335,
author = {Fukumizu, Kenji and Bach, Francis R. and Jordan, Michael I.},
title = {Dimensionality Reduction for Supervised Learning with Reproducing Kernel Hilbert Spaces},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {We propose a novel method of dimensionality reduction for supervised learning problems. Given a regression or classification problem in which we wish to predict a response variable Y from an explanatory variable X, we treat the problem of dimensionality reduction as that of finding a low-dimensional "effective subspace" for X which retains the statistical relationship between X and Y. We show that this problem can be formulated in terms of conditional independence. To turn this formulation into an optimization problem we establish a general nonparametric characterization of conditional independence using covariance operators on reproducing kernel Hilbert spaces. This characterization allows us to derive a contrast function for estimation of the effective subspace. Unlike many conventional methods for dimensionality reduction in supervised learning, the proposed method requires neither assumptions on the marginal distribution of X, nor a parametric model of the conditional distribution of Y. We present experiments that compare the performance of the method with conventional methods.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {73–99},
numpages = {27}
}

@article{10.5555/1005332.1005334,
author = {Lanckriet, Gert R. G. and Cristianini, Nello and Bartlett, Peter and Ghaoui, Laurent El and Jordan, Michael I.},
title = {Learning the Kernel Matrix with Semidefinite Programming},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {Kernel-based learning algorithms work by embedding the data into a Euclidean space, and then searching for linear relations among the embedded data points. The embedding is performed implicitly, by specifying the inner products between each pair of points in the embedding space. This information is contained in the so-called kernel matrix, a symmetric and positive semidefinite matrix that encodes the relative positions of all points. Specifying this matrix amounts to specifying the geometry of the embedding space and inducing a notion of similarity in the input space---classical model selection problems in machine learning. In this paper we show how the kernel matrix can be learned from data via semidefinite programming (SDP) techniques. When applied to a kernel matrix associated with both training and test data this gives a powerful transductive algorithm---using the labeled part of the data one can learn an embedding also for the unlabeled part. The similarity between test points is inferred from training points and their labels. Importantly, these learning problems are convex, so we obtain a method for learning both the model class and the function without local minima. Furthermore, this approach leads directly to a convex method for learning the 2-norm soft margin parameter in support vector machines, solving an important open problem.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {27–72},
numpages = {46}
}

@article{10.5555/1005332.1005333,
author = {Even-Dar, Eyal and Mansour, Yishay},
title = {Learning Rates for Q-Learning},
year = {2004},
issue_date = {12/1/2004},
publisher = {JMLR.org},
volume = {5},
issn = {1532-4435},
abstract = {In this paper we derive convergence rates for Q-learning. We show an interesting relationship between the convergence rate and the learning rate used in Q-learning. For a polynomial learning rate, one which is 1/tω at time t where ω∈(1/2,1), we show that the convergence rate is polynomial in 1/(1-γ), where γ is the discount factor. In contrast we show that for a linear learning rate, one which is 1/t at time t, the convergence rate has an exponential dependence on 1/(1-γ). In addition we show a simple example that proves this exponential behavior is inherent for linear learning rates.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1–25},
numpages = {25}
}

@article{10.1162/jmlr.2003.4.7-8.1499,
author = {Stainvas, Inna and Lowe, David},
title = {A Generative Model for Separating Illumination and Reflectance from Images},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1499},
doi = {10.1162/jmlr.2003.4.7-8.1499},
abstract = {It is well known that even slight changes in nonuniform illumination lead to a large image variability and are crucial for many visual tasks. This paper presents a new ICA related probabilistic model where the number of sources exceeds the number of sensors to perform an image segmentation and illumination removal, simultaneously. We model illumination and reflectance in log space by a generalized autoregressive process and Hidden Gaussian Markov random field, respectively.The model ability to deal with segmentation of illuminated images is compared with a Canny edge detector and homomorphic filtering. We apply the model to two problems: synthetic image segmentation and sea surface pollution detection from intensity images.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1499–1519},
numpages = {21},
keywords = {general autoregressive model, segmentation, potts model, illumination, reflectance}
}

@article{10.1162/jmlr.2003.4.7-8.1471,
author = {Bounkong, St\'{e}phane and Toch, Bor\'{e}mi and Saad, David and Lowe, David},
title = {ICA for Watermarking Digital Images},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1471},
doi = {10.1162/jmlr.2003.4.7-8.1471},
abstract = {We present a domain-independent ICA-based approach to watermarking. This approach can be used on images, music or video to embed either a robust or fragile watermark. In the case of robust watermarking, the method shows high information rate and robustness against malicious and nonmalicious attacks, while keeping a low induced distortion. The fragile watermarking scheme, on the other hand, shows high sensitivity to tampering attempts while keeping the requirement for high information rate and low distortion. The improved performance is achieved by employing a set of statistically independent sources (the independent components) as the feature space and principled statistical decoding methods. The performance of the suggested method is compared to other state of the art approaches. The paper focuses on applying the method to digitized images although the same approach can be used for other media, such as music or video.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1471–1498},
numpages = {28},
keywords = {watermarking, ICA, authentication, information-hiding, steganography}
}

@article{10.1162/jmlr.2003.4.7-8.1447,
author = {S\"{a}rel\"{a}, Jaakko and Vig\'{a}rio, Ricardo},
title = {Overlearning in Marginal Distribution-Based ICA: Analysis and Solutions},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1447},
doi = {10.1162/jmlr.2003.4.7-8.1447},
abstract = {The present paper is written as a word of caution, with users of independent component analysis (ICA) in mind, to overlearning phenomena that are often observed.We consider two types of overlearning, typical to high-order statistics based ICA. These algorithms can be seen to maximise the negentropy of the source estimates. The first kind of overlearning results in the generation of spike-like signals, if there are not enough samples in the data or there is a considerable amount of noise present. It is argued that, if the data has power spectrum characterised by 1/f curve, we face a more severe problem, which cannot be solved inside the strict ICA model. This overlearning is better characterised by bumps instead of spikes. Both overlearning types are demonstrated in the case of artificial signals as well as magnetoencephalograms (MEG). Several methods are suggested to circumvent both types, either by making the estimation of the ICA model more robust or by including further modelling of the data.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1447–1469},
numpages = {23},
keywords = {overlearning, spikes, independent component analysis, blind source separation, overfitting, bumps, high-order ICA}
}

@article{10.1162/jmlr.2003.4.7-8.1411,
author = {Waheed, Khurram and Salem, Fathi M.},
title = {Blind Source Recovery: A Framework in the State Space},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1411},
doi = {10.1162/jmlr.2003.4.7-8.1411},
abstract = {Blind Source Recovery (BSR) denotes recovery of original sources/signals from environments that may include convolution, temporal variation, and even nonlinearity. It also infers the recovery of sources even in the absence of precise environment identifiability. This paper describes, in a comprehensive fashion, a generalized BSR formulation achieved by the application of stochastic optimization principles to the Kullback-Liebler divergence as a performance functional subject to the constraints of the general (i.e., nonlinear and time-varying) state space representation. This technique is used to derive update laws for nonlinear time-varying dynamical systems, which are subsequently specialized to time-invariant and linear systems. Further, the state space demixing network structures have been exploited to develop learning rules, capable of handling most filtering paradigms, which can be conveniently extended to nonlinear models. In the special cases, distinct linear state-space algorithms are presented for the minimum phase and non-minimum phase mixing environment models. Conventional (FIR/IIR) filtering models are subsequently derived from this general structure and are compared with material in the recent literature. Illustrative simulation examples are presented to demonstrate the online adaptation capabilities of the developed algorithms. Some of this reported work has also been implemented in dedicated hardware/software platforms.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1411–1446},
numpages = {36},
keywords = {Kullback-Liebler divergence, feedforward network, feedback network, blind source recovery, state space representation}
}

@article{10.1162/jmlr.2003.4.7-8.1393,
author = {Basalyga, Gleb and Rattray, Magnus},
title = {Statistical Dynamics of On-Line Independent Component Analysis},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1393},
doi = {10.1162/jmlr.2003.4.7-8.1393},
abstract = {The learning dynamics of on-line independent component analysis is analysed in the limit of large data dimension. We study a simple Hebbian learning algorithm that can be used to separate out a small number of non-Gaussian components from a high-dimensional data set. The de-mixing matrix parameters are confined to a Stiefel manifold of tall, orthogonal matrices and we introduce a natural gradient variant of the algorithm which is appropriate to learning on this manifold. For large input dimension the parameter trajectory of both algorithms passes through a sequence of unstable fixed points, each described by a diffusion process in a polynomial potential. Choosing the learning rate too large increases the escape time from each of these fixed points, effectively trapping the learning in a sub-optimal state. In order to avoid these trapping states a very low learning rate must be chosen during the learning transient, resulting in learning time-scales of O(N2) or O(N3) iterations where N is the data dimension. Escape from each sub-optimal state results in a sequence of symmetry breaking events as the algorithm learns each source in turn. This is in marked contrast to the learning dynamics displayed by related on-line learning algorithms for multilayer neural networks and principal component analysis. Although the natural gradient variant of the algorithm has nice asymptotic convergence properties, it has an equivalent transient dynamics to the standard Hebbian algorithm.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1393–1410},
numpages = {18},
keywords = {independent component analysis, Hebbian learning, statistical mechanics, diffusion, natural gradient}
}

@article{10.1162/jmlr.2003.4.7-8.1365,
author = {Jang, Gil-Jin and Lee, Te-Won},
title = {A Maximum Likelihood Approach to Single-Channel Source Separation},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1365},
doi = {10.1162/jmlr.2003.4.7-8.1365},
abstract = {This paper presents a new technique for achieving blind signal separation when given only a single channel recording. The main concept is based on exploiting a prior sets of time-domain basis functions learned by independent component analysis (ICA) to the separation of mixed source signals observed in a single channel. The inherent time structure of sound sources is reflected in the ICA basis functions, which encode the sources in a statistically efficient manner. We derive a learning algorithm using a maximum likelihood approach given the observed single channel data and sets of basis functions. For each time point we infer the source parameters and their contribution factors. This inference is possible due to prior knowledge of the basis functions and the associated coefficient densities. A flexible model for density estimation allows accurate modeling of the observation and our experimental results exhibit a high level of separation performance for simulated mixtures as well as real environment recordings employing mixtures of two different sources.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1365–1392},
numpages = {28},
keywords = {independent component analysis (ICA), computational auditory scene analysis (CASA), blind signal separation (BSS), sparse coding, generalized Gaussian distribution}
}

@article{10.1162/jmlr.2003.4.7-8.1339,
author = {Kisilev, Pavel and Zibulevsky, Michael and Zeevi, Yehoshua Y.},
title = {A Multiscale Framework for Blind Separation of Linearly Mixed Signals},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1339},
doi = {10.1162/jmlr.2003.4.7-8.1339},
abstract = {We consider the problem of blind separation of unknown source signals or images from a given set of their linear mixtures. It was discovered recently that exploiting the sparsity of sources and their mixtures, once they are projected onto a proper space of sparse representation, improves the quality of separation. In this study we take advantage of the properties of multiscale transforms, such as wavelet packets, to decompose signals into sets of local features with various degrees of sparsity. We then study how the separation error is affected by the sparsity of decomposition coefficients, and by the misfit between the probabilistic model of these coefficients and their actual distribution. Our error estimator, based on the Taylor expansion of the quasi-ML function, is used in selection of the best subsets of coefficients and utilized, in turn, in further separation. The performance of the algorithm is evaluated by using noise-free and noisy data. Experiments with simulated signals, musical sounds and images, demonstrate significant improvement of separation quality over previously reported results.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1339–1364},
numpages = {26},
keywords = {maximum likelihood, wavelets, blind source separation, multiscale transforms}
}

@article{10.1162/jmlr.2003.4.7-8.1319,
author = {Ziehe, Andreas and Kawanabe, Motoaki and Harmeling, Stefan and M\"{u}ller, Klaus-Robert},
title = {Blind Separation of Post-Nonlinear Mixtures Using Linearizing Transformations and Temporal Decorrelation},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1319},
doi = {10.1162/jmlr.2003.4.7-8.1319},
abstract = {We propose two methods that reduce the post-nonlinear blind source separation problem (PNL-BSS) to a linear BSS problem. The first method is based on the concept of maximal correlation: we apply the alternating conditional expectation (ACE) algorithm--a powerful technique from non-parametric statistics--to approximately invert the componentwise nonlinear functions. The second method is a Gaussianizing transformation, which is motivated by the fact that linearly mixed signals before nonlinear transformation are approximately Gaussian distributed. This heuristic, but simple and efficient procedure works as good as the ACE method. Using the framework provided by ACE, convergence can be proven. The optimal transformations obtained by ACE coincide with the sought-after inverse functions of the nonlinearities. After equalizing the nonlinearities, temporal decorrelation separation (TDSEP) allows us to recover the source signals. Numerical simulations testing "ACE-TD" and "Gauss-TD" on realistic examples are performed with excellent results.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1319–1338},
numpages = {20},
keywords = {maximal correlation, temporal decorrelation, blind source separation, Gaussianization, post-nonlinear mixture}
}

@article{10.1162/jmlr.2003.4.7-8.1297,
author = {Almeida, Lu\'{\i}s B.},
title = {MISEP - Linear and Nonlinear ICA Based on Mutual Information},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1297},
doi = {10.1162/jmlr.2003.4.7-8.1297},
abstract = {Linear Independent Components Analysis (ICA) has become an important signal processing and data analysis technique, the typical application being blind source separation in a wide range of signals, such as biomedical, acoustical and astrophysical ones. Nonlinear ICA is less developed, but has the potential to become at least as powerful.This paper presents MISEP, an ICA technique for linear and nonlinear mixtures, which is based on the minimization of the mutual information of the estimated components. MISEP is a generalization of the popular INFOMAX technique, which is extended in two ways: (1) to deal with nonlinear mixtures, and (2) to be able to adapt to the actual statistical distributions of the sources, by dynamically estimating the nonlinearities to be used at the outputs. The resulting MISEP method optimizes a network with a specialized architecture, with a single objective function: the output entropy.The paper also briefly discusses the issue of nonlinear source separation. Examples of linear and nonlinear source separation performed by MISEP are presented.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1297–1318},
numpages = {22},
keywords = {nonlinear ICA, ICA, blind source separation, mutual information}
}

@article{10.1162/jmlr.2003.4.7-8.1271,
author = {Learned-Miller, Erik G. and Fisher, John W.},
title = {ICA Using Spacings Estimates of Entropy},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1271},
doi = {10.1162/jmlr.2003.4.7-8.1271},
abstract = {This paper presents a new algorithm for the independent components analysis (ICA) problem based on an efficient entropy estimator. Like many previous methods, this algorithm directly minimizes the measure of departure from independence according to the estimated Kullback-Leibler divergence between the joint distribution and the product of the marginal distributions. We pair this approach with efficient entropy estimators from the statistics literature. In particular, the entropy estimator we use is consistent and exhibits rapid convergence. The algorithm based on this estimator is simple, computationally efficient, intuitively appealing, and outperforms other well known algorithms. In addition, the estimator's relative insensitivity to outliers translates into superior performance by our ICA algorithm on outlier tests. We present favorable comparisons to the Kernel ICA, FAST-ICA, JADE, and extended Infomax algorithms in extensive simulations. We also provide public domain source code for our algorithms.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1271–1295},
numpages = {25}
}

@article{10.1162/jmlr.2003.4.7-8.1261,
author = {Parra, Lucas and Sajda, Paul},
title = {Blind Source Separation via Generalized Eigenvalue Decomposition},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1261},
doi = {10.1162/jmlr.2003.4.7-8.1261},
abstract = {In this short note we highlight the fact that linear blind source separation can be formulated as a generalized eigenvalue decomposition under the assumptions of non-Gaussian, non-stationary, or non-white independent sources. The solution for the unmixing matrix is given by the generalized eigenvectors that simultaneously diagonalize the covariance matrix of the observations and an additional symmetric matrix whose form depends upon the particular assumptions. The method critically determines the mixture coefficients and is therefore not robust to estimation errors. However it provides a rather general and unified solution that summarizes the conditions for successful blind source separation. To demonstrate the method, which can be implemented in two lines of matlab code, we present results for artificial mixtures of speech and real mixtures of electroencephalography (EEG) data, showing that the same sources are recovered under the various assumptions.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1261–1269},
numpages = {9},
keywords = {non-stationary, non-Gaussian, generalized eigenvalue decomposition, blind source separation, non-white}
}

@article{10.1162/jmlr.2003.4.7-8.1235,
author = {Teh, Yee Whye and Welling, Max and Osindero, Simon and Hinton, Geoffrey E.},
title = {Energy-Based Models for Sparse Overcomplete Representations},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1235},
doi = {10.1162/jmlr.2003.4.7-8.1235},
abstract = {We present a new way of extending independent components analysis (ICA) to overcomplete representations. In contrast to the causal generative extensions of ICA which maintain marginal independence of sources, we define features as deterministic (linear) functions of the inputs. This assumption results in marginal dependencies among the features, but conditional independence of the features given the inputs. By assigning energies to the features a probability distribution over the input states is defined through the Boltzmann distribution. Free parameters of this model are trained using the contrastive divergence objective (Hinton, 2002). When the number of features is equal to the number of input dimensions this energy-based model reduces to noiseless ICA and we show experimentally that the proposed learning algorithm is able to perform blind source separation on speech data. In additional experiments we train overcomplete energy-based models to extract features from various standard data-sets containing speech, natural images, hand-written digits and faces.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1235–1260},
numpages = {26},
keywords = {overcomplete representations, independent components analysis, sparse representations, density estimation}
}

@article{10.1162/jmlr.2003.4.7-8.1205,
author = {Bach, Francis R. and Jordan, Michael I.},
title = {Beyond Independent Components: Trees and Clusters},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1205},
doi = {10.1162/jmlr.2003.4.7-8.1205},
abstract = {We present a generalization of independent component analysis (ICA), where instead of looking for a linear transform that makes the data components independent, we look for a transform that makes the data components well fit by a tree-structured graphical model. This tree-dependent component analysis (TCA) provides a tractable and flexible approach to weakening the assumption of independence in ICA. In particular, TCA allows the underlying graph to have multiple connected components, and thus the method is able to find "clusters" of components such that components are dependent within a cluster and independent between clusters. Finally, we make use of a notion of graphical models for time series due to Brillinger (1996) to extend these ideas to the temporal setting. In particular, we are able to fit models that incorporate tree-structured dependencies among multiple time series.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1205–1233},
numpages = {29},
keywords = {independent component analysis, time series, graphical models, semiparametric models, blind source separation}
}

@article{10.1162/jmlr.2003.4.7-8.1177,
author = {Cardoso, Jean-Fran\c{c}ois},
title = {Dependence, Correlation and Gaussianity in Independent Component Analysis},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1177},
doi = {10.1162/jmlr.2003.4.7-8.1177},
abstract = {Independent component analysis (ICA) is the decomposition of a random vector in linear components which are "as independent as possible." Here, "independence" should be understood in its strong statistical sense: it goes beyond (second-order) decorrelation and thus involves the non-Gaussianity of the data. The ideal measure of independence is the "mutual information" and is known to be related to the entropy of the components when the search for components is restricted to uncorrelated components. This paper explores the connections between mutual information, entropy and non-Gaussianity in a larger framework, without resorting to a somewhat arbitrary decorrelation constraint. A key result is that the mutual information can be decomposed, under linear transforms, as the sum of two terms: one term expressing the decorrelation of the components and one expressing their non-Gaussianity.Our results extend the previous understanding of these connections and explain them in the light of information geometry. We also describe the "local geometry" of ICA by re-expressing all our results via a Gram-Charlier expansion by which all quantities of interest are obtained in terms of cumulants.},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1177–1203},
numpages = {27},
keywords = {source separation, minimum entropy, non-Gaussianity, mutual information, cumulant expansions, independent component analysis, information geometry}
}

@article{10.1162/jmlr.2003.4.7-8.1175,
author = {Lee, Te-Won and Cardoso, Jean-Fran\c{c}is and Oja, Erkki and Amari, Shun-ichi},
title = {Introduction to Special Issue on Independent Components Analysis},
year = {2004},
issue_date = {October 1 - November 15, 2004},
publisher = {JMLR.org},
volume = {4},
number = {7–8},
issn = {1532-4435},
url = {https://doi.org/10.1162/jmlr.2003.4.7-8.1175},
doi = {10.1162/jmlr.2003.4.7-8.1175},
journal = {J. Mach. Learn. Res.},
month = oct,
pages = {1175–1176},
numpages = {2}
}

@article{10.5555/945365.964315,
author = {Stainvas, Inna and Lowe, David},
title = {A Generative Model for Separating Illumination and Reflectance from Images},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {It is well known that even slight changes in nonuniform illumination lead to a large image variability and are crucial for many visual tasks. This paper presents a new ICA related probabilistic model where the number of sources exceeds the number of sensors to perform an image segmentation and illumination removal, simultaneously. We model illumination and reflectance in log space by a generalized autoregressive process and Hidden Gaussian Markov random field, respectively.The model ability to deal with segmentation of illuminated images is compared with a Canny edge detector and homomorphic filtering. We apply the model to two problems: synthetic image segmentation and sea surface pollution detection from intensity images.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1499–1519},
numpages = {21}
}

@article{10.5555/945365.964314,
author = {Bounkong, St\'{e}phane and Toch, Bor\'{e}mi and Saad, David and Lowe, David},
title = {ICA for Watermarking Digital Images},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We present a domain-independent ICA-based approach to watermarking. This approach can be used on images, music or video to embed either a robust or fragile watermark.In the case of robust watermarking, the method shows high information rate and robustness against malicious and non-malicious attacks, while keeping a low induced distortion. The fragile watermarking scheme, on the other hand, shows high sensitivity to tampering attempts while keeping the requirement for high information rate and low distortion. The improved performance is achieved by employing a set of statistically independent sources (the independent components) as the feature space and principled statistical decoding methods. The performance of the suggested method is compared to other state of the art approaches. The paper focuses on applying the method to digitized images although the same approach can be used for other media, such as music or video.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1471–1498},
numpages = {28}
}

@article{10.5555/945365.964313,
author = {S\"{a}rel\"{a}, Jaakko and Vig\'{a}rio, Ricardo},
title = {Overlearning in Marginal Distribution-Based ICA: Analysis and Solutions},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {The present paper is written as a word of caution, with users of independent component analysis (ICA) in mind, to overlearning phenomena that are often observed.We consider two types of overlearning, typical to high-order statistics based ICA. These algorithms can be seen to maximise the negentropy of the source estimates. The first kind of overlearning results in the generation of spike-like signals, if there are not enough samples in the data or there is a considerable amount of noise present. It is argued that, if the data has power spectrum characterised by 1/f curve, we face a more severe problem, which cannot be solved inside the strict ICA model. This overlearning is better characterised by bumps instead of spikes. Both overlearning types are demonstrated in the case of artificial signals as well as magnetoencephalograms (MEG). Several methods are suggested to circumvent both types, either by making the estimation of the ICA model more robust or by including further modelling of the data.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1447–1469},
numpages = {23}
}

@article{10.5555/945365.964312,
author = {Waheed, Khurram and Salem, Fathi M.},
title = {Blind Source Recovery: A Framework in the State Space},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Blind Source Recovery (BSR) denotes recovery of original
sources/signals from environments that may include convolution,
temporal variation, and even nonlinearity. It also infers the
recovery of sources even in the absence of precise environment
identifiability. This paper describes, in a comprehensive fashion,
a generalized BSR formulation achieved by the application of
stochastic optimization principles to the Kullback-Liebler
divergence as a performance functional subject to the constraints
of the general (i.e., nonlinear and time-varying) state space
representation. This technique is used to derive update laws for
nonlinear time-varying dynamical systems, which are subsequently
specialized to time-invariant and linear systems. Further, the
state space demixing network structures have been exploited to
develop learning rules, capable of handling most filtering
paradigms, which can be conveniently extended to nonlinear models.
In the special cases, distinct linear state-space algorithms are
presented for the minimum phase and non-minimum phase mixing
environment models. Conventional (FIR/IIR) filtering models are
subsequently derived from this general structure and are compared
with material in the recent literature. Illustrative simulation
examples are presented to demonstrate the online adaptation
capabilities of the developed algorithms. Some of this reported
work has also been implemented in dedicated hardware/software
platforms.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1411–1446},
numpages = {36}
}

@article{10.5555/945365.964311,
author = {Basalyga, Gleb and Rattray, Magnus},
title = {Statistical Dynamics of On-Line Independent Component Analysis},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {The learning dynamics of on-line independent component analysis is analysed in the limit of large data dimension. We study a simple Hebbian learning algorithm that can be used to separate out a small number of non-Gaussian components from a high-dimensional data set. The de-mixing matrix parameters are confined to a Stiefel manifold of tall, orthogonal matrices and we introduce a natural gradient variant of the algorithm which is appropriate to learning on this manifold. For large input dimension the parameter trajectory of both algorithms passes through a sequence of unstable fixed points, each described by a diffusion process in a polynomial potential. Choosing the learning rate too large increases the escape time from each of these fixed points, effectively trapping the learning in a sub-optimal state. In order to avoid these trapping states a very low learning rate must be chosen during the learning transient, resulting in learning time-scales of O(N2) or O(N3) iterations where N is the data dimension. Escape from each sub-optimal state results in a sequence of symmetry breaking events as the algorithm learns each source in turn. This is in marked contrast to the learning dynamics displayed by related on-line learning algorithms for multilayer neural networks and principal component analysis. Although the natural gradient variant of the algorithm has nice asymptotic convergence properties, it has an equivalent transient dynamics to the standard Hebbian algorithm.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1393–1410},
numpages = {18}
}

@article{10.5555/945365.964310,
author = {Jang, Gil-Jin and Lee, Te-Won},
title = {A Maximum Likelihood Approach to Single-Channel Source Separation},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {This paper presents a new technique for achieving blind signal
separation when given only a single channel recording. The main
concept is based on exploiting a priori sets of time-domain
basis functions learned by independent component analysis (ICA) to
the separation of mixed source signals observed in a single
channel. The inherent time structure of sound sources is reflected
in the ICA basis functions, which encode the sources in a
statistically efficient manner. We derive a learning algorithm
using a maximum likelihood approach given the observed single
channel data and sets of basis functions. For each time point we
infer the source parameters and their contribution factors. This
inference is possible due to prior knowledge of the basis functions
and the associated coefficient densities. A flexible model for
density estimation allows accurate modeling of the observation and
our experimental results exhibit a high level of separation
performance for simulated mixtures as well as real environment
recordings employing mixtures of two different sources.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1365–1392},
numpages = {28}
}

@article{10.5555/945365.964309,
author = {Kisilev, Pavel and Zibulevsky, Michael and Zeevi, Yehoshua Y.},
title = {A Multiscale Framework for Blind Separation of Linearly Mixed Signals},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We consider the problem of blind separation of unknown source signals or images from a given set of their linear mixtures. It was discovered recently that exploiting the sparsity of sources and their mixtures, once they are projected onto a proper space of sparse representation, improves the quality of separation. In this study we take advantage of the properties of multiscale transforms, such as wavelet packets, to decompose signals into sets of local features with various degrees of sparsity. We then study how the separation error is affected by the sparsity of decomposition coefficients, and by the misfit between the probabilistic model of these coefficients and their actual distribution. Our error estimator, based on the Taylor expansion of the quasi-ML function, is used in selection of the best subsets of coefficients and utilized, in turn, in further separation. The performance of the algorithm is evaluated by using noise-free and noisy data. Experiments with simulated signals, musical sounds and images, demonstrate significant improvement of separation quality over previously reported results.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1339–1363},
numpages = {25}
}

@article{10.5555/945365.964308,
author = {Ziehe, Andreas and Kawanabe, Motoaki and Harmeling, Stefan and M\"{u}ller, Klaus-Robert},
title = {Blind Separation of Post-Nonlinear Mixtures Using Linearizing Transformations and Temporal Decorrelation},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We propose two methods that reduce the post-nonlinear blind source
separation problem (PNL-BSS) to a linear BSS problem. The first
method is based on the concept of maximal correlation: we
apply the alternating conditional expectation (ACE) algorithm---a
powerful technique from non-parametric statistics---to
approximately invert the componentwise non-linear functions.The
second method is a Gaussianizing transformation, which is motivated
by the fact that linearly mixed signals before nonlinear
transformation are approximately Gaussian distributed. This
heuristic, but simple and efficient procedure works as good as the
ACE method.Using the framework provided by ACE, convergence can be
proven. The optimal transformations obtained by ACE coincide with
the sought-after inverse functions of the nonlinearities. After
equalizing the nonlinearities, temporal decorrelation separation
(TDSEP) allows us to recover the source signals. Numerical
simulations testing "ACE-TD" and "Gauss-TD" on realistic examples
are performed with excellent results.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1319–1338},
numpages = {20}
}

@article{10.5555/945365.964307,
author = {Almeida, Lu\'{\i}s B.},
title = {Misep—Linear and Nonlinear ICA Based on Mutual Information},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Linear Independent Components Analysis (ICA) has become an important signal processing and data analysis technique, the typical application being blind source separation in a wide range of signals, such as biomedical, acoustical and astrophysical ones. Nonlinear ICA is less developed, but has the potential to become at least as powerful.This paper presents MISEP, an ICA technique for linear and nonlinear mixtures, which is based on the minimization of the mutual information of the estimated components. MISEP is a generalization of the popular INFOMAX technique, which is extended in two ways: (1) to deal with nonlinear mixtures, and (2) to be able to adapt to the actual statistical distributions of the sources, by dynamically estimating the nonlinearities to be used at the outputs. The resulting MISEP method optimizes a network with a specialized architecture, with a single objective function: the output entropy.The paper also briefly discusses the issue of nonlinear source separation. Examples of linear and nonlinear source separation performed by MISEP are presented.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1297–1318},
numpages = {22}
}

@article{10.5555/945365.964306,
author = {Learned-Miller, Erik G. and III, John W. Fisher},
title = {ICA Using Spacings Estimates of Entropy},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {This paper presents a new algorithm for the independent components analysis (ICA) problem based on an efficient entropy estimator. Like many previous methods, this algorithm directly minimizes the measure of departure from independence according to the estimated Kullback-Leibler divergence between the joint distribution and the product of the marginal distributions. We pair this approach with efficient entropy estimators from the statistics literature. In particular, the entropy estimator we use is consistent and exhibits rapid convergence. The algorithm based on this estimator is simple, computationally efficient, intuitively appealing, and outperforms other well known algorithms. In addition, the estimator's relative insensitivity to outliers translates into superior performance by our ICA algorithm on outlier tests. We present favorable comparisons to the Kernel ICA, FAST-ICA, JADE, and extended Infomax algorithms in extensive simulations. We also provide public domain source code for our algorithms.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1271–1295},
numpages = {25}
}

@article{10.5555/945365.964305,
author = {Parra, Lucas and Sajda, Paul},
title = {Blind Source Separation via Generalized Eigenvalue Decomposition},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {In this short note we highlight the fact that linear blind source separation can be formulated as a generalized eigenvalue decomposition under the assumptions of non-Gaussian, non-stationary, or non-white independent sources. The solution for the unmixing matrix is given by the generalized eigenvectors that simultaneously diagonalize the covariance matrix of the observations and an additional symmetric matrix whose form depends upon the particular assumptions. The method critically determines the mixture coefficients and is therefore not robust to estimation errors. However it provides a rather general and unified solution that summarizes the conditions for successful blind source separation. To demonstrate the method, which can be implemented in two lines of matlab code, we present results for artificial mixtures of speech and real mixtures of electroencephalography (EEG) data, showing that the same sources are recovered under the various assumptions.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1261–1269},
numpages = {9}
}

@article{10.5555/945365.964304,
author = {Teh, Yee Whye and Welling, Max and Osindero, Simon and Hinton, Geoffrey E.},
title = {Energy-Based Models for Sparse Overcomplete Representations},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We present a new way of extending independent components analysis
(ICA) to overcomplete representations. In contrast to the causal
generative extensions of ICA which maintain marginal independence
of sources, we define features as deterministic
(linear) functions of the inputs. This assumption results in
marginal dependencies among the features, but conditional
independence of the features given the inputs. By assigning
energies to the features a probability distribution over the input
states is defined through the Boltzmann distribution. Free
parameters of this model are trained using the contrastive
divergence objective (Hinton, 2002). When the number of features is
equal to the number of input dimensions this energy-based model
reduces to noiseless ICA and we show experimentally that the
proposed learning algorithm is able to perform blind source
separation on speech data. In additional experiments we train
overcomplete energy-based models to extract features from various
standard data-sets containing speech, natural images, hand-written
digits and faces.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1235–1260},
numpages = {26}
}

@article{10.5555/945365.964303,
author = {Bach, Francis R. and Jordan, Michael I.},
title = {Beyond Independent Components: Trees and Clusters},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We present a generalization of independent component analysis
(ICA), where instead of looking for a linear transform that makes
the data components independent, we look for a transform that makes
the data components well fit by a tree-structured graphical model.
This tree-dependent component analysis (TCA) provides a
tractable and flexible approach to weakening the assumption of
independence in ICA. In particular, TCA allows the underlying graph
to have multiple connected components, and thus the method is able
to find "clusters" of components such that components are dependent
within a cluster and independent between clusters. Finally, we make
use of a notion of graphical models for time series due to
Brillinger (1996) to extend these ideas to the temporal setting. In
particular, we are able to fit models that incorporate
tree-structured dependencies among multiple time series.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1205–1233},
numpages = {29}
}

@article{10.5555/945365.964302,
author = {Cardoso, Jean-Fran\c{c}ois},
title = {Dependence, Correlation and Gaussianity in Independent Component Analysis},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Independent component analysis (ICA) is the decomposition of a random vector in linear components which are "as independent as possible." Here, "independence" should be understood in its strong statistical sense: it goes beyond (second-order) decorrelation and thus involves the non-Gaussianity of the data.The ideal measure of independence is the "mutual information" and is known to be related to the entropy of the components when the search for components is restricted to uncorrelated components.This paper explores the connections between mutual information, entropy and non-Gaussianity in a larger framework, without resorting to a somewhat arbitrary decorrelation constraint. A key result is that the mutual information can be decomposed, under linear transforms, as the sum of two terms: one term expressing the decorrelation of the components and one expressing their non-Gaussianity.Our results extend the previous understanding of these connections and explain them in the light of information geometry. We also describe the "local geometry" of ICA by re-expressing all our results via a Gram-Charlier expansion by which all quantities of interest are obtained in terms of cumulants.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1177–1203},
numpages = {27}
}

@article{10.5555/945365.964301,
author = {Lee, Te-Won and Cardoso, Jean-Fran\c{c}ois and Oja, Erkki and Amari, Shun-ichi},
title = {Introduction to Special Issue on Independent Components Analysis},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1175–1176},
numpages = {2}
}

@article{10.5555/945365.964299,
author = {McAllester, David and Ortiz, Luis},
title = {Concentration Inequalities for the Missing Mass and for Histogram Rule Error},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {This paper gives distribution-free concentration inequalities for the missing mass and the error rate of histogram rules. Negative association methods can be used to reduce these concentration problems to concentration questions about independent sums. Although the sums are independent, they are highly heterogeneous. Such highly heterogeneous independent sums cannot be analyzed using standard concentration inequalities such as Hoeffding's inequality, the Angluin-Valiant bound, Bernstein's inequality, Bennett's inequality, or McDiarmid's theorem. The concentration inequality for histogram rule error is motivated by the desire to construct a new class of bounds on the generalization error of decision trees.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {895–911},
numpages = {17}
}

@article{10.5555/945365.964298,
author = {Blanchard, Gilles and Lugosi, G´bor and Vayatis, Nicolas},
title = {On the Rate of Convergence of Regularized Boosting Classifiers},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {A regularized boosting method is introduced, for which regularization is obtained through a penalization function. It is shown through oracle inequalities that this method is model adaptive. The rate of convergence of the probability of misclassification is investigated. It is shown that for quite a large class of distributions, the probability of error converges to the Bayes risk at a rate faster than n-(V+2)/(4(V+1)) where V is the VC dimension of the "base" class whose elements are combined by boosting methods to obtain an aggregated classifier. The dimension-independent nature of the rates may partially explain the good behavior of these methods in practical problems. Under Tsybakov's noise condition the rate of convergence is even faster. We investigate the conditions necessary to obtain such rates for different base classes. The special case of boosting using decision stumps is studied in detail. We characterize the class of classifiers realizable by aggregating decision stumps. It is shown that some versions of boosting work especially well in high-dimensional logistic additive models. It appears that adding a limited labelling noise to the training data may in certain cases improve the convergence, as has been also suggested by other authors.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {861–894},
numpages = {34}
}

@article{10.5555/945365.964297,
author = {Meir, Ron and Zhang, Tong},
title = {Generalization Error Bounds for Bayesian Mixture Algorithms},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Bayesian approaches to learning and estimation have played a significant role in the Statistics literature over many years. While they are often provably optimal in a frequentist setting, and lead to excellent performance in practical applications, there have not been many precise characterizations of their performance for finite sample sizes under general conditions. In this paper we consider the class of Bayesian mixture algorithms, where an estimator is formed by constructing a data-dependent mixture over some hypothesis space. Similarly to what is observed in practice, our results demonstrate that mixture approaches are particularly robust, and allow for the construction of highly complex estimators, while avoiding undesirable overfitting effects. Our results, while being data-dependent in nature, are insensitive to the underlying model assumptions, and apply whether or not these hold. At a technical level, the approach applies to unbounded functions, constrained only by certain moment conditions. Finally, the bounds derived can be directly applied to non-Bayesian mixture approaches such as Boosting and Bagging.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {839–860},
numpages = {22}
}

@article{10.5555/945365.964296,
author = {Mesterharm, Chris},
title = {Tracking Linear-Threshold Concepts with Winnow},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {In this paper, we give a mistake-bound for learning arbitrary linear-threshold concepts that are allowed to change over time in the on-line model of learning. We use a variation of the Winnow algorithm and show that the bounds for learning shifting linear-threshold functions have many of the same advantages that the traditional Winnow algorithm has on fixed concepts. These benefits include a weak dependence on the number of irrelevant attributes, inexpensive runtime, and robust behavior against noise. In fact, we show that the bound for tracking Winnow has even better performance with respect to irrelevant attributes. Let X∈[0,1]n be an instance of the learning problem. In the previous bounds, the number of mistakes depends on lnn. In this paper, the shifting concept bound depends on max ln(||X||1). We show that this behavior is a result of certain parameter choices in the tracking version of Winnow, and we show how to use related parameters to get a similar mistake bound for the traditional fixed concept version of Winnow.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {819–838},
numpages = {20}
}

@article{10.5555/945365.964295,
author = {Takimoto, Eiji and Warmuth, Manfred K.},
title = {Path Kernels and Multiplicative Updates},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Kernels are typically applied to linear algorithms whose weight vector is a linear combination of the feature vectors of the examples. On-line versions of these algorithms are sometimes called "additive updates" because they add a multiple of the last feature vector to the current weight vector.In this paper we have found a way to use special convolution kernels to efficiently implement "multiplicative" updates. The kernels are defined by a directed graph. Each edge contributes an input. The inputs along a path form a product feature and all such products build the feature vector associated with the inputs.We also have a set of probabilities on the edges so that the outflow from each vertex is one. We then discuss multiplicative updates on these graphs where the prediction is essentially a kernel computation and the update contributes a factor to each edge. After adding the factors to the edges, the total outflow out of each vertex is not one any more. However some clever algorithms re-normalize the weights on the paths so that the total outflow out of each vertex is one again. Finally, we show that if the digraph is built from a regular expressions, then this can be used for speeding up the kernel and re-normalization computations.We reformulate a large number of multiplicative update algorithms using path kernels and characterize the applicability of our method. The examples include efficient algorithms for learning disjunctions and a recent algorithm that predicts as well as the best pruning of a series parallel digraphs.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {773–818},
numpages = {46}
}

@article{10.5555/945365.964294,
author = {Mendelson, Shahar},
title = {On the Performance of Kernel Classes},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We present sharp bounds on the localized Rademacher averages of the unit ball in a reproducing kernel Hilbert space in terms of the eigenvalues of the integral operator associated with the kernel. We use this result to estimate the performance of the empirical minimization algorithm when the base class is the unit ball of the reproducing kernel Hilbert space.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {759–771},
numpages = {13}
}

@article{10.5555/945365.964293,
author = {Herbrich, Ralf and Graepel, Thore},
title = {Introduction to the Special Issue on Learning Theory},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {755–757},
numpages = {3}
}

@article{10.5555/945365.964291,
author = {Malzahn, D\"{o}rthe and Opper, Manfred},
title = {An Approximate Analytical Approach to Resampling Averages},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Using a novel reformulation, we develop a framework to compute approximate resampling data averages analytically. The method avoids multiple retraining of statistical models on the samples. Our approach uses a combination of the replica "trick" of statistical physics and the TAP approach for approximate Bayesian inference. We demonstrate our approach on regression with Gaussian processes. A comparison with averages obtained by Monte-Carlo sampling shows that our method achieves good accuracy.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1151–1173},
numpages = {23}
}

@article{10.5555/945365.964290,
author = {Lagoudakis, Michail G. and Parr, Ronald},
title = {Least-Squares Policy Iteration},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We propose a new approach to reinforcement learning for control problems which combines value-function approximation with linear architectures and approximate policy iteration. This new approach is motivated by the least-squares temporal-difference learning algorithm (LSTD) for prediction problems, which is known for its efficient use of sample experiences compared to pure temporal-difference algorithms. Heretofore, LSTD has not had a straightforward application to control problems mainly because LSTD learns the state value function of a fixed policy which cannot be used for action selection and control without a model of the underlying process. Our new algorithm, least-squares policy iteration (LSPI), learns the state-action value function which allows for action selection without a model and for incremental policy improvement within a policy-iteration framework. LSPI is a model-free, off-policy method which can use efficiently (and reuse in each iteration) sample experiences collected in any manner. By separating the sample collection method, the choice of the linear approximation architecture, and the solution method, LSPI allows for focused attention on the distinct elements that contribute to practical reinforcement learning. LSPI is tested on the simple task of balancing an inverted pendulum and the harder task of balancing and riding a bicycle to a target location. In both cases, LSPI learns to control the pendulum or the bicycle by merely observing a relatively small number of trials where actions are selected randomly. LSPI is also compared against Q-learning (both with and without experience replay) using the same value function architecture. While LSPI achieves good performance fairly consistently on the difficult bicycle task, Q-learning variants were rarely able to balance for more than a small fraction of the time needed to reach the target location.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1107–1149},
numpages = {43}
}

@article{10.5555/945365.964289,
author = {Steinwart, Ingo},
title = {Sparseness of Support Vector Machines},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Support vector machines (SVMs) construct decision functions that are linear combinations of kernel evaluations on the training set. The samples with non-vanishing coefficients are called support vectors. In this work we establish lower (asymptotical) bounds on the number of support vectors. On our way we prove several results which are of great importance for the understanding of SVMs. In particular, we describe to which "limit" SVM decision functions tend, discuss the corresponding notion of convergence and provide some results on the stability of SVMs using subdifferential calculus in the associated reproducing kernel Hilbert space.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1071–1105},
numpages = {35}
}

@article{10.5555/945365.964288,
author = {Hu, Junling and Wellman, Michael P.},
title = {Nash Q-Learning for General-Sum Stochastic Games},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We extend Q-learning to a noncooperative multiagent context, using the framework of general-sum stochastic games. A learning agent maintains Q-functions over joint actions, and performs updates based on assuming Nash equilibrium behavior over the current Q-values. This learning protocol provably converges given certain restrictions on the stage games (defined by Q-values) that arise during learning. Experiments with a pair of two-player grid games suggest that such restrictions on the game structure are not necessarily required. Stage games encountered during learning in both grid environments violate the conditions. However, learning consistently converges in the first grid game, which has a unique equilibrium Q-function, but sometimes fails to converge in the second, which has three different equilibrium Q-functions. In a comparison of offline learning performance in both games, we find agents are more likely to reach a joint optimal path with Nash Q-learning than with a single-agent Q-learning method. When at least one agent adopts Nash Q-learning, the performance of both agents is better than using single-agent Q-learning. We have also implemented an online version of Nash Q-learning that balances exploration with exploitation, yielding improved performance.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1039–1069},
numpages = {31}
}

@article{10.5555/945365.964287,
author = {Zhong, Shi and Ghosh, Joydeep},
title = {A Unified Framework for Model-Based Clustering},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Model-based clustering techniques have been widely used and have shown promising results in many applications involving complex data. This paper presents a unified framework for probabilistic model-based clustering based on a bipartite graph view of data and models that highlights the commonalities and differences among existing model-based clustering algorithms. In this view, clusters are represented as probabilistic models in a model space that is conceptually separate from the data space. For partitional clustering, the view is conceptually similar to the Expectation-Maximization (EM) algorithm. For hierarchical clustering, the graph-based view helps to visualize critical/important distinctions between similarity-based approaches and model-based approaches. The framework also suggests several useful variations of existing clustering algorithms. Two new variations---balanced model-based clustering and hybrid model-based clustering---are discussed and empirically evaluated on a variety of data types.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {1001–1037},
numpages = {37}
}

@article{10.5555/945365.964286,
author = {Hutter, Marcus},
title = {Optimality of Universal Bayesian Sequence Prediction for General Loss and Alphabet},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Various optimality properties of universal sequence predictors based on Bayes-mixtures in general, and Solomonoff's prediction scheme in particular, will be studied.The probability of observing xt at time t, given past observations x1...xt-1 can be computed with the chain rule if the true generating distribution μ of the sequences x1x2x3.... is known. If μ is unknown, but known to belong to a countable or continuous class M one can base ones prediction on the Bayes-mixture ξ defined as a wν-weighted sum or integral of distributions ν ∈ M. The cumulative expected loss of the Bayes-optimal universal prediction scheme based on ξ is shown to be close to the loss of the Bayes-optimal, but infeasible prediction scheme based on μ. We show that the bounds are tight and that no other predictor can lead to significantly smaller bounds.Furthermore, for various performance measures, we show Pareto-optimality of ξ and give an Occam's razor argument that the choice wν ∼ 2-K(ν) for the weights is optimal, where K(ν) is the length of the shortest program describing ν.The results are applied to games of chance, defined as a sequence of bets, observations, and rewards.The prediction schemes (and bounds) are compared to the popular predictors based on expert advice.Extensions to infinite alphabets, partial, delayed and probabilistic prediction, classification, and more active systems are briefly discussed.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {971–1000},
numpages = {30}
}

@article{10.5555/945365.964285,
author = {Freund, Yoav and Iyer, Raj and Schapire, Robert E. and Singer, Yoram},
title = {An Efficient Boosting Algorithm for Combining Preferences},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We study the problem of learning to accurately rank a set of objects by combining a given collection of ranking or preference functions. This problem of combining preferences arises in several applications, such as that of combining the results of different search engines, or the "collaborative-filtering" problem of ranking movies for a user based on the movie rankings provided by other users. In this work, we begin by presenting a formal framework for this general problem. We then describe and analyze an efficient algorithm called RankBoost for combining preferences based on the boosting approach to machine learning. We give theoretical results describing the algorithm's behavior both on the training data, and on new test data not seen during training. We also describe an efficient implementation of the algorithm for a particular restricted but common case. We next discuss two experiments we carried out to assess the performance of RankBoost. In the first experiment, we used the algorithm to combine different web search strategies, each of which is a query expansion for a given domain. The second experiment is a collaborative-filtering task for making movie recommendations.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {933–969},
numpages = {37}
}

@article{10.5555/945365.964284,
author = {Wolf, Lior and Shashua, Amnon},
title = {Learning over Sets Using Kernel Principal Angles},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We consider the problem of learning with instances defined over a space of sets of vectors. We derive a new positive definite kernel f(A,B) defined over pairs of matrices A,B based on the concept of principal angles between two linear subspaces. We show that the principal angles can be recovered using only inner-products between pairs of column vectors of the input matrices thereby allowing the original column vectors of A,B to be mapped onto arbitrarily high-dimensional feature spaces.We demonstrate the usage of the matrix-based kernel function f(A,B) with experiments on two visual tasks. The first task is the discrimination of "irregular" motion trajectory of an individual or a group of individuals in a video sequence. We use the SVM approach using f(A,B) where an input matrix represents the motion trajectory of a group of individuals over a certain (fixed) time frame. We show that the classification (irregular versus regular) greatly outperforms the conventional representation where all the trajectories form a single vector. The second application is the visual recognition of faces from input video sequences representing head motion and facial expressions where f(A,B) is used to compare two image sequences.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {913–931},
numpages = {19}
}

@article{10.5555/945365.945393,
author = {Claveau, Vincent and S\'{e}billot, Pascale and Fabre, C\'{e}cile and Bouillon, Pierrette},
title = {Learning Semantic Lexicons from a Part-of-Speech and Semantically Tagged Corpus Using Inductive Logic Programming},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {This paper describes an inductive logic programming learning method designed to acquire from a corpus specific Noun-Verb (N-V) pairs---relevant in information retrieval applications to perform index expansion---in order to build up semantic lexicons based on Pustejovsky's generative lexicon (GL) principles (Pustejovsky, 1995). In one of the components of this lexical model, called the <em>qualia structure</em>, words are described in terms of semantic roles. For example, the <em>telic</em> role indicates the purpose or function of an item (<em>cut</em> for <em>knife</em>), the agentive role its creation mode (<em>build</em> for <em>house</em>), etc. The qualia structure of a noun is mainly made up of verbal associations, encoding relational information. The learning method enables us to automatically extract, from a morpho-syntactically and semantically tagged corpus, N-V pairs whose elements are linked by one of the semantic relations defined in the qualia structure in GL. It also infers rules explaining what in the surrounding context distinguishes such pairs from others also found in sentences of the corpus but which are not relevant. Stress is put here on the learning efficiency that is required to be able to deal with all the available contextual information, and to produce linguistically meaningful rules.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {493–525},
numpages = {33}
}

@article{10.5555/945365.945392,
author = {Costa, V\'{\i}tor Santos and Srinivasan, Ashwin and Camacho, Rui and Blockeel, Hendrik and Demoen, Bart and Janssens, Gerda and Struyf, Jan and Vandecasteele, Henk and Laer, Wim Van},
title = {Query Transformations for Improving the Efficiency of Ilp Systems},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Relatively simple transformations can speed up the execution of queries for data mining considerably. While some ILP systems use such transformations, relatively little is known about them or how they relate to each other. This paper describes a number of such transformations. Not all of them are novel, but there have been no studies comparing their efficacy. The main contributions of the paper are: (a) it clarifies the relationship between the transformations; (b) it contains an empirical study of what can be gained by applying the transformations; and (c) it provides some guidance on the kinds of problems that are likely to benefit from the transformations.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {465–491},
numpages = {27}
}

@article{10.5555/945365.945391,
author = {Botta, Marco and Giordana, Attilio and Saitta, Lorenza and Sebag, Mich\`{e}le},
title = {Relational Learning as Search in a Critical Region},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Machine learning strongly relies on the covering test to assess whether a candidate hypothesis covers training examples. The present paper investigates learning relational concepts from examples, termed <em>relational learning</em> or <em>inductive logic programming</em>. In particular, it investigates the chances of success and the computational cost of relational learning, which appears to be severely affected by the presence of a phase transition in the covering test. To this aim, three up-to-date relational learners have been applied to a wide range of artificial, fully relational learning problems. A first experimental observation is that the phase transition behaves as an attractor for relational learning; no matter which region the learning problem belongs to, all three learners produce hypotheses lying within or close to the phase transition region. Second, a <em>failure region</em> appears. All three learners fail to learn any accurate hypothesis in this region. Quite surprisingly, the probability of failure does not systematically increase with the size of the underlying target concept: under some circumstances, longer concepts may be easier to accurately approximate than shorter ones. Some interpretations for these findings are proposed and discussed.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {431–463},
numpages = {33}
}

@article{10.5555/945365.945390,
author = {Page, David and Srinivasan, Ashwin},
title = {Ilp: A Short Look Back and a Longer Look Forward},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Inductive logic programming (ILP) is built on a foundation laid by research in machine learning and computational logic. Armed with this strong foundation, ILP has been applied to important and interesting problems in the life sciences, engineering and the arts. This paper begins by briefly reviewing some example applications, in order to illustrate the benefits of ILP. In turn, the applications have brought into focus the need for more research into specific topics. We enumerate and elaborate five of these: (1) novel search methods; (2) incorporation of explicit probabilities; (3) incorporation of special-purpose reasoners; (4) parallel execution using commodity components; and (5) enhanced human interaction. It is our hypothesis that progress in each of these areas can greatly improve the contributions that can be made with ILP; and that, with assistance from research workers in other areas, significant progress in each of these areas is possible.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {415–430},
numpages = {16}
}

@article{10.5555/945365.945389,
author = {Cussens, James and Frisch, Alan M.},
title = {Introduction to the Special Issue on Inductive Logic Programming},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {413–414},
numpages = {2}
}

@article{10.5555/945365.945387,
author = {Srinivasan, Ashwin and King, Ross D. and Bain, Michael E.},
title = {An Empirical Study of the Use of Relevance Information in Inductive Logic Programming},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Inductive Logic Programming (ILP) systems construct models for data using domain-specific background information. When using these systems, it is typically assumed that sufficient human expertise is at hand to rule out irrelevant background information. Such irrelevant information can, and typically does, hinder an ILP system's search for good models. Here, we provide evidence that if expertise is available that can provide a partial-ordering on sets of background predicates in terms of relevance to the analysis task, then this can be used to good effect by an ILP system. In particular, using data from biochemical domains, we investigate an incremental strategy of including sets of predicates in decreasing order of relevance. Results obtained suggest that: (a) the incremental approach identifies, in substantially less time, a model that is comparable in predictive accuracy to that obtained with all background information in place; and (b) the incremental approach using the relevance ordering performs better than one that does not (that is, one that adds sets of predicates randomly). For a practitioner concerned with use of ILP, the implication of these findings are two-fold: (1) when not all background information can be used at once (either due to limitations of the ILP system, or the nature of the domain) expert assessment of the relevance of background predicates can assist substantially in the construction of good models; and (2) good "first-cut" results can be obtained quickly by a simple exclusion of information known to be less relevant.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {369–383},
numpages = {15}
}

@article{10.5555/945365.945386,
author = {Langseth, Helge and Nielsen, Thomas D.},
title = {Fusion of Domain Knowledge with Data for Structural Learning in Object Oriented Domains},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {When constructing a Bayesian network, it can be advantageous to employ structural learning algorithms to combine knowledge captured in databases with prior information provided by domain experts. Unfortunately, conventional learning algorithms do not easily incorporate prior information, if this information is too vague to be encoded as properties that are local to families of variables. For instance, conventional algorithms do not exploit prior information about repetitive structures, which are often found in object oriented domains such as computer networks, large pedigrees and genetic analysis. In this paper we propose a method for doing structural learning in object oriented domains. It is demonstrated that this method is more efficient than conventional algorithms in such domains, and it is argued that the method supports a natural approach for expressing and incorporating prior information provided by domain experts.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {339–368},
numpages = {30}
}

@article{10.5555/945365.945385,
author = {Haddawy, Peter and Ha, Vu and Restificar, Angelo and Geisler, Benjamin and Miyamoto, John},
title = {Preference Elicitation via Theory Refinement},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We present an approach to elicitation of user preference models in which assumptions can be used to guide but not constrain the elicitation process. We demonstrate that when domain knowledge is available, even in the form of weak and somewhat inaccurate assumptions, significantly less data is required to build an accurate model of user preferences than when no domain knowledge is provided. This approach is based on the KBANN (Knowledge-Based Artificial Neural Network) algorithm pioneered by Shavlik and Towell (1989). We demonstrate this approach through two examples, one involves preferences under certainty, and the other involves preferences under uncertainty. In the case of certainty, we show how to encode assumptions concerning preferential independence and monotonicity in a KBANN network, which can be trained using a variety of preferential information including simple binary classification. In the case of uncertainty, we show how to construct a KBANN network that encodes certain types of dominance relations and attitude toward risk. The resulting network can be trained using answers to standard gamble questions and can be used as an approximate representation of a person's preferences. We empirically evaluate our claims by comparing the KBANN networks with simple backpropagation artificial neural networks in terms of learning rate and accuracy. For the case of uncertainty, the answers to standard gamble questions used in the experiment are taken from an actual medical data set first used by Miyamoto and Eraker (1988). In the case of certainty, we define a measure to which a set of preferences violate a domain theory, and examine the robustness of the KBANN network as this measure of domain theory violation varies.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {317–337},
numpages = {21}
}

@article{10.5555/945365.945384,
author = {Druzdzel, Marek J. and D\'{\i}ez, Francisco J.},
title = {Combining Knowledge from Different Sources in Causal Probabilistic Models},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {Building probabilistic and decision-theoretic models requires a considerable knowledge engineering effort in which the most daunting task is obtaining the numerical parameters. Authors of Bayesian networks usually combine various sources of information, such as textbooks, statistical reports, databases, and expert judgement. In this paper, we demonstrate the risks of such a combination, even when this knowledge encompasses such seemingly population-independent characteristics as sensitivity and specificity of medical symptoms. We show that the criteria ``do not combine knowledge from different sources'' or ``use only data from the setting in which the model will be used'' are neither necessary nor sufficient to guarantee the correctness of the model. Instead, we offer graphical criteria for determining when knowledge from different sources can be safely combined into the general population model. We also offer a method for building subpopulation models. The analysis performed in this paper and the criteria we propose may be useful in such fields as knowledge engineering, epidemiology, machine learning, and statistical meta-analysis.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {295–316},
numpages = {22}
}

@article{10.5555/945365.945383,
author = {Dybowski, Richard and Laskey, Kathryn B. and Myers, James W. and Parsons, Simon},
title = {Introduction to the Special Issue on the Fusion of Domain Knowledge with Data for Decision Support},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {293–294},
numpages = {2}
}

@article{10.5555/945365.945377,
author = {Gadanho, Sandra Clara},
title = {Learning Behavior-Selection by Emotions and Cognition in a Multi-Goal Robot Task},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {The existence of emotion and cognition as two interacting systems, both with important roles in decision-making, has been recently advocated by neurophysiological research (LeDoux, 1998, Damasio, 1994. Following that idea, this paper presents the ALEC agent architecture which has both emotive and cognitive learning, as well as emotive and cognitive decision-making capabilities to adapt to real-world environments. These two learning mechanisms embody very different properties which can be related to those of natural emotion and cognition systems. The reported experiments test ALEC within a simulated autonomous robot which learns to perform a multi-goal and multi-step survival task when faced with real world conditions, namely continuous time and space, noisy sensors and unreliable actuators. Experimental results show that both systems contribute positively to the learning performance of the agent.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {385–412},
numpages = {28}
}

@article{10.5555/945365.945376,
author = {Friedman, Craig and Sandow, Sven},
title = {Learning Probabilistic Models: An Expected Utility Maximization Approach},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
abstract = {We consider the problem of learning a probabilistic model from the viewpoint of an expected utility maximizing decision maker/investor who would use the model to make decisions (bets), which result in well defined payoffs. In our new approach, we seek good out-of-sample model performance by considering a one-parameter family of Pareto optimal models, which we define in terms of consistency with the training data and consistency with a prior (benchmark) model. We measure the former by means of the large-sample distribution of a vector of sample-averaged features, and the latter by means of a generalized relative entropy. We express each Pareto optimal model as the solution of a strictly convex optimization problem and its strictly concave (and tractable) dual. Each dual problem is a regularized maximization of expected utility over a well-defined family of functions. Each Pareto optimal model is robust: maximizing worst-case outperformance relative to the benchmark model. Finally, we select the Pareto optimal model with maximum (out-of-sample) expected utility. We show that our method reduces to the minimum relative entropy method if and only if the utility function is a member of a three-parameter logarithmic family.},
journal = {J. Mach. Learn. Res.},
month = dec,
pages = {257–291},
numpages = {35}
}

@article{10.1162/153244304773936117,
author = {d'Avignon, Christian and Geman, Donald},
title = {Tree-Structured Neural Decoding},
year = {2003},
issue_date = {12/1/2003},
publisher = {JMLR.org},
volume = {4},
number = {null},
issn = {1532-4435},
url = {https://doi.org/10.1162/153244304773936117},
doi = {10.1162/153244304773936117},
abstract = {We propose adaptive testing as a general

license: Unlicense

repository: goyalkaraniit/acm-cr

repo_name: goyalkaraniit/acm-cr